2024-09-17T11:40:26.471 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-17T11:40:26.489 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-17T11:40:26.575 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-16_20:08:15-orch-main-distro-default-smithi/7907794 branch: main description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: ceph-qa@ceph.com first_in_suite: false job_id: '7907794' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-09-16_20:08:15-orch-main-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: main ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_REFRESH_FAILED log-only-match: - CEPHADM_ sha1: 7498493279982dfe87f00616198a5967475f6169 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 7498493279982dfe87f00616198a5967475f6169 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: main sha1: 2b698643d05f73e79c2d2888228947a84a4ac0bd owner: scheduled_teuthology@teuthology priority: 950 repo: https://git.ceph.com/ceph.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 3649 sha1: 7498493279982dfe87f00616198a5967475f6169 sleep_before_teardown: 0 subset: 42/64 suite: orch suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_2b698643d05f73e79c2d2888228947a84a4ac0bd/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 2b698643d05f73e79c2d2888228947a84a4ac0bd targets: smithi045.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMWgftta3OzhWNUtNpBFBap9GFVWLMmXwneyPJK/FQDr96G0UYsg18vgxZVuTPWmtaaeNXtkWXh0cJT0+6FHX2g= smithi165.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFOotCXS/rhY85M5NqDHE25+FTP/W115m+TxioPhXQT8czaHOaqNPnRfDnlJuYmslxf4+e9JhUyKldmSzB8LXe8= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-16_20:08:15 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-09-17T11:40:26.576 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_2b698643d05f73e79c2d2888228947a84a4ac0bd/qa; will attempt to use it 2024-09-17T11:40:26.576 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_2b698643d05f73e79c2d2888228947a84a4ac0bd/qa/tasks 2024-09-17T11:40:26.577 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-17T11:40:26.578 INFO:teuthology.task.internal:Checking packages... 2024-09-17T11:40:26.599 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '7498493279982dfe87f00616198a5967475f6169' 2024-09-17T11:40:26.599 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-17T11:40:26.599 INFO:teuthology.packaging:ref: None 2024-09-17T11:40:26.599 INFO:teuthology.packaging:tag: None 2024-09-17T11:40:26.599 INFO:teuthology.packaging:branch: main 2024-09-17T11:40:26.599 INFO:teuthology.packaging:sha1: 7498493279982dfe87f00616198a5967475f6169 2024-09-17T11:40:26.600 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=main 2024-09-17T11:40:26.842 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-4959.g2b698643 2024-09-17T11:40:26.843 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-17T11:40:26.851 INFO:teuthology.task.internal:no buildpackages task found 2024-09-17T11:40:26.851 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-17T11:40:26.865 INFO:teuthology.task.internal:Saving configuration 2024-09-17T11:40:26.882 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-17T11:40:26.889 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-17T11:40:26.913 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi045.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-16_20:08:15-orch-main-distro-default-smithi/7907794', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-17 11:35:26.950291', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMWgftta3OzhWNUtNpBFBap9GFVWLMmXwneyPJK/FQDr96G0UYsg18vgxZVuTPWmtaaeNXtkWXh0cJT0+6FHX2g='} 2024-09-17T11:40:26.938 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi165.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-16_20:08:15-orch-main-distro-default-smithi/7907794', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-17 11:35:26.948522', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFOotCXS/rhY85M5NqDHE25+FTP/W115m+TxioPhXQT8czaHOaqNPnRfDnlJuYmslxf4+e9JhUyKldmSzB8LXe8='} 2024-09-17T11:40:26.938 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-17T11:40:26.944 INFO:teuthology.task.internal:roles: ubuntu@smithi045.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-09-17T11:40:26.944 INFO:teuthology.task.internal:roles: ubuntu@smithi165.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-09-17T11:40:26.944 INFO:teuthology.run_tasks:Running task console_log... 2024-09-17T11:40:27.010 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fa7ffb4fac0>, signals=[15]) 2024-09-17T11:40:27.011 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-17T11:40:27.017 INFO:teuthology.task.internal:Opening connections... 2024-09-17T11:40:27.017 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi045.front.sepia.ceph.com 2024-09-17T11:40:27.018 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi045.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T11:40:27.098 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi165.front.sepia.ceph.com 2024-09-17T11:40:27.098 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi165.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T11:40:27.173 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-17T11:40:27.179 DEBUG:teuthology.orchestra.run.smithi045:> uname -m 2024-09-17T11:40:27.197 INFO:teuthology.orchestra.run.smithi045.stdout:x86_64 2024-09-17T11:40:27.197 DEBUG:teuthology.orchestra.run.smithi045:> cat /etc/os-release 2024-09-17T11:40:27.254 INFO:teuthology.orchestra.run.smithi045.stdout:NAME="CentOS Stream" 2024-09-17T11:40:27.254 INFO:teuthology.orchestra.run.smithi045.stdout:VERSION="9" 2024-09-17T11:40:27.254 INFO:teuthology.orchestra.run.smithi045.stdout:ID="centos" 2024-09-17T11:40:27.254 INFO:teuthology.orchestra.run.smithi045.stdout:ID_LIKE="rhel fedora" 2024-09-17T11:40:27.254 INFO:teuthology.orchestra.run.smithi045.stdout:VERSION_ID="9" 2024-09-17T11:40:27.254 INFO:teuthology.orchestra.run.smithi045.stdout:PLATFORM_ID="platform:el9" 2024-09-17T11:40:27.255 INFO:teuthology.orchestra.run.smithi045.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-17T11:40:27.255 INFO:teuthology.orchestra.run.smithi045.stdout:ANSI_COLOR="0;31" 2024-09-17T11:40:27.255 INFO:teuthology.orchestra.run.smithi045.stdout:LOGO="fedora-logo-icon" 2024-09-17T11:40:27.255 INFO:teuthology.orchestra.run.smithi045.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-17T11:40:27.255 INFO:teuthology.orchestra.run.smithi045.stdout:HOME_URL="https://centos.org/" 2024-09-17T11:40:27.255 INFO:teuthology.orchestra.run.smithi045.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-17T11:40:27.255 INFO:teuthology.orchestra.run.smithi045.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-17T11:40:27.255 INFO:teuthology.orchestra.run.smithi045.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-17T11:40:27.256 INFO:teuthology.lock.ops:Updating smithi045.front.sepia.ceph.com on lock server 2024-09-17T11:40:27.274 DEBUG:teuthology.orchestra.run.smithi165:> uname -m 2024-09-17T11:40:27.292 INFO:teuthology.orchestra.run.smithi165.stdout:x86_64 2024-09-17T11:40:27.292 DEBUG:teuthology.orchestra.run.smithi165:> cat /etc/os-release 2024-09-17T11:40:27.349 INFO:teuthology.orchestra.run.smithi165.stdout:NAME="CentOS Stream" 2024-09-17T11:40:27.349 INFO:teuthology.orchestra.run.smithi165.stdout:VERSION="9" 2024-09-17T11:40:27.349 INFO:teuthology.orchestra.run.smithi165.stdout:ID="centos" 2024-09-17T11:40:27.350 INFO:teuthology.orchestra.run.smithi165.stdout:ID_LIKE="rhel fedora" 2024-09-17T11:40:27.350 INFO:teuthology.orchestra.run.smithi165.stdout:VERSION_ID="9" 2024-09-17T11:40:27.350 INFO:teuthology.orchestra.run.smithi165.stdout:PLATFORM_ID="platform:el9" 2024-09-17T11:40:27.350 INFO:teuthology.orchestra.run.smithi165.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-17T11:40:27.350 INFO:teuthology.orchestra.run.smithi165.stdout:ANSI_COLOR="0;31" 2024-09-17T11:40:27.350 INFO:teuthology.orchestra.run.smithi165.stdout:LOGO="fedora-logo-icon" 2024-09-17T11:40:27.350 INFO:teuthology.orchestra.run.smithi165.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-17T11:40:27.350 INFO:teuthology.orchestra.run.smithi165.stdout:HOME_URL="https://centos.org/" 2024-09-17T11:40:27.350 INFO:teuthology.orchestra.run.smithi165.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-17T11:40:27.350 INFO:teuthology.orchestra.run.smithi165.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-17T11:40:27.350 INFO:teuthology.orchestra.run.smithi165.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-17T11:40:27.351 INFO:teuthology.lock.ops:Updating smithi165.front.sepia.ceph.com on lock server 2024-09-17T11:40:27.374 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-17T11:40:27.382 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-17T11:40:27.391 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-17T11:40:27.391 DEBUG:teuthology.orchestra.run.smithi045:> test '!' -e /home/ubuntu/cephtest 2024-09-17T11:40:27.393 DEBUG:teuthology.orchestra.run.smithi165:> test '!' -e /home/ubuntu/cephtest 2024-09-17T11:40:27.409 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-17T11:40:27.415 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-17T11:40:27.415 DEBUG:teuthology.orchestra.run.smithi045:> test -z $(ls -A /var/lib/ceph) 2024-09-17T11:40:27.450 DEBUG:teuthology.orchestra.run.smithi165:> test -z $(ls -A /var/lib/ceph) 2024-09-17T11:40:27.553 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-17T11:40:27.609 INFO:teuthology.run_tasks:Running task kernel... 2024-09-17T11:40:27.624 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-17T11:40:27.624 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-17T11:40:27.624 DEBUG:teuthology.orchestra.run.smithi045:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-17T11:40:27.625 DEBUG:teuthology.orchestra.run.smithi165:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-17T11:40:27.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T11:40:27.640 DEBUG:teuthology.orchestra.run.smithi165:> uname -r 2024-09-17T11:40:27.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T11:40:27.642 DEBUG:teuthology.orchestra.run.smithi045:> uname -r 2024-09-17T11:40:27.696 INFO:teuthology.orchestra.run.smithi165.stdout:5.14.0-503.el9.x86_64 2024-09-17T11:40:27.696 INFO:teuthology.task.kernel:Running kernel on smithi165: 5.14.0-503.el9.x86_64 2024-09-17T11:40:27.696 DEBUG:teuthology.orchestra.run.smithi165:> sudo yum install -y kernel 2024-09-17T11:40:27.699 INFO:teuthology.orchestra.run.smithi045.stdout:5.14.0-503.el9.x86_64 2024-09-17T11:40:27.699 INFO:teuthology.task.kernel:Running kernel on smithi045: 5.14.0-503.el9.x86_64 2024-09-17T11:40:27.699 DEBUG:teuthology.orchestra.run.smithi045:> sudo yum install -y kernel 2024-09-17T11:40:31.279 INFO:teuthology.orchestra.run.smithi165.stdout:CentOS Stream 9 - BaseOS 4.3 MB/s | 8.3 MB 00:01 2024-09-17T11:40:31.449 INFO:teuthology.orchestra.run.smithi045.stdout:CentOS Stream 9 - BaseOS 4.1 MB/s | 8.3 MB 00:02 2024-09-17T11:40:33.071 INFO:teuthology.orchestra.run.smithi165.stdout:CentOS Stream 9 - AppStream 29 MB/s | 20 MB 00:00 2024-09-17T11:40:34.542 INFO:teuthology.orchestra.run.smithi045.stdout:CentOS Stream 9 - AppStream 10 MB/s | 20 MB 00:01 2024-09-17T11:40:38.597 INFO:teuthology.orchestra.run.smithi165.stdout:CentOS Stream 9 - CRB 3.8 MB/s | 6.5 MB 00:01 2024-09-17T11:40:40.005 INFO:teuthology.orchestra.run.smithi165.stdout:CentOS Stream 9 - Extras packages 106 kB/s | 19 kB 00:00 2024-09-17T11:40:40.108 INFO:teuthology.orchestra.run.smithi045.stdout:CentOS Stream 9 - CRB 3.9 MB/s | 6.5 MB 00:01 2024-09-17T11:40:41.109 INFO:teuthology.orchestra.run.smithi165.stdout:Extra Packages for Enterprise Linux 23 MB/s | 23 MB 00:00 2024-09-17T11:40:42.384 INFO:teuthology.orchestra.run.smithi045.stdout:CentOS Stream 9 - Extras packages 18 kB/s | 19 kB 00:01 2024-09-17T11:40:43.179 INFO:teuthology.orchestra.run.smithi045.stdout:Extra Packages for Enterprise Linux 34 MB/s | 23 MB 00:00 2024-09-17T11:40:46.515 INFO:teuthology.orchestra.run.smithi165.stdout:lab-extras 32 kB/s | 1.7 kB 00:00 2024-09-17T11:40:48.422 INFO:teuthology.orchestra.run.smithi165.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-17T11:40:48.422 INFO:teuthology.orchestra.run.smithi165.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-17T11:40:48.506 INFO:teuthology.orchestra.run.smithi165.stdout:Dependencies resolved. 2024-09-17T11:40:48.517 INFO:teuthology.orchestra.run.smithi165.stdout:================================================================================ 2024-09-17T11:40:48.517 INFO:teuthology.orchestra.run.smithi165.stdout: Package Architecture Version Repository Size 2024-09-17T11:40:48.517 INFO:teuthology.orchestra.run.smithi165.stdout:================================================================================ 2024-09-17T11:40:48.517 INFO:teuthology.orchestra.run.smithi165.stdout:Installing: 2024-09-17T11:40:48.517 INFO:teuthology.orchestra.run.smithi165.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-17T11:40:48.517 INFO:teuthology.orchestra.run.smithi165.stdout:Installing dependencies: 2024-09-17T11:40:48.517 INFO:teuthology.orchestra.run.smithi165.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-17T11:40:48.518 INFO:teuthology.orchestra.run.smithi165.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-17T11:40:48.518 INFO:teuthology.orchestra.run.smithi165.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-17T11:40:48.518 INFO:teuthology.orchestra.run.smithi165.stdout: 2024-09-17T11:40:48.518 INFO:teuthology.orchestra.run.smithi165.stdout:Transaction Summary 2024-09-17T11:40:48.518 INFO:teuthology.orchestra.run.smithi165.stdout:================================================================================ 2024-09-17T11:40:48.518 INFO:teuthology.orchestra.run.smithi165.stdout:Install 4 Packages 2024-09-17T11:40:48.518 INFO:teuthology.orchestra.run.smithi165.stdout: 2024-09-17T11:40:48.518 INFO:teuthology.orchestra.run.smithi165.stdout:Total download size: 79 M 2024-09-17T11:40:48.519 INFO:teuthology.orchestra.run.smithi165.stdout:Installed size: 126 M 2024-09-17T11:40:48.519 INFO:teuthology.orchestra.run.smithi165.stdout:Downloading Packages: 2024-09-17T11:40:48.677 INFO:teuthology.orchestra.run.smithi045.stdout:lab-extras 48 kB/s | 1.7 kB 00:00 2024-09-17T11:40:49.366 INFO:teuthology.orchestra.run.smithi165.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 63 kB/s | 36 kB 00:00 2024-09-17T11:40:50.191 INFO:teuthology.orchestra.run.smithi165.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 11 MB/s | 16 MB 00:01 2024-09-17T11:40:50.658 INFO:teuthology.orchestra.run.smithi045.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-17T11:40:50.659 INFO:teuthology.orchestra.run.smithi045.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-17T11:40:50.750 INFO:teuthology.orchestra.run.smithi045.stdout:Dependencies resolved. 2024-09-17T11:40:50.754 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-17T11:40:50.754 INFO:teuthology.orchestra.run.smithi045.stdout: Package Architecture Version Repository Size 2024-09-17T11:40:50.754 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-17T11:40:50.754 INFO:teuthology.orchestra.run.smithi045.stdout:Installing: 2024-09-17T11:40:50.754 INFO:teuthology.orchestra.run.smithi045.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-17T11:40:50.755 INFO:teuthology.orchestra.run.smithi045.stdout:Installing dependencies: 2024-09-17T11:40:50.755 INFO:teuthology.orchestra.run.smithi045.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-17T11:40:50.755 INFO:teuthology.orchestra.run.smithi045.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-17T11:40:50.755 INFO:teuthology.orchestra.run.smithi045.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-17T11:40:50.755 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:40:50.755 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction Summary 2024-09-17T11:40:50.755 INFO:teuthology.orchestra.run.smithi045.stdout:================================================================================ 2024-09-17T11:40:50.755 INFO:teuthology.orchestra.run.smithi045.stdout:Install 4 Packages 2024-09-17T11:40:50.755 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:40:50.756 INFO:teuthology.orchestra.run.smithi045.stdout:Total download size: 79 M 2024-09-17T11:40:50.756 INFO:teuthology.orchestra.run.smithi045.stdout:Installed size: 126 M 2024-09-17T11:40:50.756 INFO:teuthology.orchestra.run.smithi045.stdout:Downloading Packages: 2024-09-17T11:40:50.817 INFO:teuthology.orchestra.run.smithi165.stdout:(3/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 17 MB/s | 35 MB 00:02 2024-09-17T11:40:50.903 INFO:teuthology.orchestra.run.smithi045.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 355 kB/s | 36 kB 00:00 2024-09-17T11:40:51.892 INFO:teuthology.orchestra.run.smithi165.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 11 MB/s | 29 MB 00:02 2024-09-17T11:40:51.892 INFO:teuthology.orchestra.run.smithi165.stdout:-------------------------------------------------------------------------------- 2024-09-17T11:40:51.892 INFO:teuthology.orchestra.run.smithi165.stdout:Total 23 MB/s | 79 MB 00:03 2024-09-17T11:40:52.053 INFO:teuthology.orchestra.run.smithi045.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 13 MB/s | 16 MB 00:01 2024-09-17T11:40:52.574 INFO:teuthology.orchestra.run.smithi165.stdout:Running transaction check 2024-09-17T11:40:52.612 INFO:teuthology.orchestra.run.smithi045.stdout:(3/4): kernel-modules-core-5.14.0-508.el9.x86_6 17 MB/s | 29 MB 00:01 2024-09-17T11:40:52.926 INFO:teuthology.orchestra.run.smithi165.stdout:Transaction check succeeded. 2024-09-17T11:40:52.926 INFO:teuthology.orchestra.run.smithi165.stdout:Running transaction test 2024-09-17T11:40:53.104 INFO:teuthology.orchestra.run.smithi045.stdout:(4/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 15 MB/s | 35 MB 00:02 2024-09-17T11:40:53.104 INFO:teuthology.orchestra.run.smithi045.stdout:-------------------------------------------------------------------------------- 2024-09-17T11:40:53.104 INFO:teuthology.orchestra.run.smithi045.stdout:Total 34 MB/s | 79 MB 00:02 2024-09-17T11:40:53.819 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction check 2024-09-17T11:40:53.899 INFO:teuthology.orchestra.run.smithi165.stdout:Transaction test succeeded. 2024-09-17T11:40:53.899 INFO:teuthology.orchestra.run.smithi165.stdout:Running transaction 2024-09-17T11:40:54.197 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction check succeeded. 2024-09-17T11:40:54.197 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction test 2024-09-17T11:40:55.208 INFO:teuthology.orchestra.run.smithi045.stdout:Transaction test succeeded. 2024-09-17T11:40:55.208 INFO:teuthology.orchestra.run.smithi045.stdout:Running transaction 2024-09-17T11:40:56.342 INFO:teuthology.orchestra.run.smithi165.stdout: Preparing : 1/1 2024-09-17T11:40:57.069 INFO:teuthology.orchestra.run.smithi165.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-17T11:40:57.151 INFO:teuthology.orchestra.run.smithi165.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T11:40:57.773 INFO:teuthology.orchestra.run.smithi045.stdout: Preparing : 1/1 2024-09-17T11:40:58.559 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-17T11:40:58.647 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T11:40:59.922 INFO:teuthology.orchestra.run.smithi165.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T11:41:00.051 INFO:teuthology.orchestra.run.smithi165.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T11:41:01.388 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T11:41:01.525 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T11:41:04.885 INFO:teuthology.orchestra.run.smithi165.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T11:41:04.918 INFO:teuthology.orchestra.run.smithi165.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T11:41:06.650 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T11:41:06.692 INFO:teuthology.orchestra.run.smithi045.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T11:41:09.784 INFO:teuthology.orchestra.run.smithi165.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T11:41:11.841 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T11:41:48.015 INFO:teuthology.orchestra.run.smithi165.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T11:41:48.055 INFO:teuthology.orchestra.run.smithi165.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-17T11:41:48.428 INFO:teuthology.orchestra.run.smithi165.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T11:41:48.428 INFO:teuthology.orchestra.run.smithi165.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-17T11:41:48.428 INFO:teuthology.orchestra.run.smithi165.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T11:41:48.428 INFO:teuthology.orchestra.run.smithi165.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T11:41:49.256 INFO:teuthology.orchestra.run.smithi165.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T11:41:49.256 INFO:teuthology.orchestra.run.smithi165.stdout: 2024-09-17T11:41:49.256 INFO:teuthology.orchestra.run.smithi165.stdout:Installed: 2024-09-17T11:41:49.257 INFO:teuthology.orchestra.run.smithi165.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-17T11:41:49.257 INFO:teuthology.orchestra.run.smithi165.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-17T11:41:49.257 INFO:teuthology.orchestra.run.smithi165.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-17T11:41:49.257 INFO:teuthology.orchestra.run.smithi165.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-17T11:41:49.257 INFO:teuthology.orchestra.run.smithi165.stdout: 2024-09-17T11:41:49.257 INFO:teuthology.orchestra.run.smithi165.stdout:Complete! 2024-09-17T11:41:49.639 DEBUG:teuthology.orchestra.run.smithi165:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T11:41:49.719 INFO:teuthology.orchestra.run.smithi165.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T11:41:49.719 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T11:41:49.719 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-17T11:41:49.720 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-17T11:41:49.720 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-17T11:41:49.720 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-09-17T11:41:49.720 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi165.front.sepia.ceph.com, path=None, version=distro) 2024-09-17T11:41:49.720 DEBUG:teuthology.orchestra.run.smithi165:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T11:41:49.778 INFO:teuthology.orchestra.run.smithi165.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T11:41:49.779 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T11:41:49.779 DEBUG:teuthology.orchestra.run.smithi165:> sudo rpm -qi grub2-tools 2024-09-17T11:41:49.811 INFO:teuthology.orchestra.run.smithi165.stdout:Name : grub2-tools 2024-09-17T11:41:49.811 INFO:teuthology.orchestra.run.smithi165.stdout:Epoch : 1 2024-09-17T11:41:49.811 INFO:teuthology.orchestra.run.smithi165.stdout:Version : 2.06 2024-09-17T11:41:49.811 INFO:teuthology.orchestra.run.smithi165.stdout:Release : 82.el9 2024-09-17T11:41:49.811 INFO:teuthology.orchestra.run.smithi165.stdout:Architecture: x86_64 2024-09-17T11:41:49.811 INFO:teuthology.orchestra.run.smithi165.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-17T11:41:49.811 INFO:teuthology.orchestra.run.smithi165.stdout:Group : Unspecified 2024-09-17T11:41:49.811 INFO:teuthology.orchestra.run.smithi165.stdout:Size : 8274670 2024-09-17T11:41:49.811 INFO:teuthology.orchestra.run.smithi165.stdout:License : GPLv3+ 2024-09-17T11:41:49.811 INFO:teuthology.orchestra.run.smithi165.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-17T11:41:49.812 INFO:teuthology.orchestra.run.smithi165.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-17T11:41:49.812 INFO:teuthology.orchestra.run.smithi165.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-17T11:41:49.812 INFO:teuthology.orchestra.run.smithi165.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-17T11:41:49.812 INFO:teuthology.orchestra.run.smithi165.stdout:Packager : builder@centos.org 2024-09-17T11:41:49.812 INFO:teuthology.orchestra.run.smithi165.stdout:Vendor : CentOS 2024-09-17T11:41:49.812 INFO:teuthology.orchestra.run.smithi165.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-17T11:41:49.812 INFO:teuthology.orchestra.run.smithi165.stdout:Summary : Support tools for GRUB. 2024-09-17T11:41:49.812 INFO:teuthology.orchestra.run.smithi165.stdout:Description : 2024-09-17T11:41:49.812 INFO:teuthology.orchestra.run.smithi165.stdout: 2024-09-17T11:41:49.812 INFO:teuthology.orchestra.run.smithi165.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-17T11:41:49.812 INFO:teuthology.orchestra.run.smithi165.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-17T11:41:49.812 INFO:teuthology.orchestra.run.smithi165.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-17T11:41:49.813 INFO:teuthology.orchestra.run.smithi165.stdout:hardware devices. 2024-09-17T11:41:49.813 INFO:teuthology.orchestra.run.smithi165.stdout: 2024-09-17T11:41:49.813 INFO:teuthology.orchestra.run.smithi165.stdout:This subpackage provides tools for support of all platforms. 2024-09-17T11:41:49.813 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-17T11:41:49.814 INFO:teuthology.task.kernel:Updating grub on smithi165 to boot 5.14.0-508.el9.x86_64 2024-09-17T11:41:49.814 DEBUG:teuthology.orchestra.run.smithi165:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-17T11:41:50.593 INFO:teuthology.orchestra.run.smithi165.stderr:Generating grub configuration file ... 2024-09-17T11:41:50.929 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T11:41:50.970 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-17T11:41:51.311 INFO:teuthology.orchestra.run.smithi045.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T11:41:51.311 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-17T11:41:51.311 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T11:41:51.311 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T11:41:52.042 INFO:teuthology.orchestra.run.smithi165.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-17T11:41:52.068 INFO:teuthology.orchestra.run.smithi165.stderr:done 2024-09-17T11:41:52.070 DEBUG:teuthology.orchestra.run.smithi165:> mktemp 2024-09-17T11:41:52.085 INFO:teuthology.orchestra.run.smithi165.stdout:/tmp/tmp.XFA9XxwtUp 2024-09-17T11:41:52.086 DEBUG:teuthology.orchestra.run.smithi165:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.XFA9XxwtUp 2024-09-17T11:41:52.153 DEBUG:teuthology.orchestra.run.smithi165:> sudo chmod 0666 /tmp/tmp.XFA9XxwtUp 2024-09-17T11:41:52.209 INFO:teuthology.orchestra.run.smithi045.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T11:41:52.210 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:41:52.210 INFO:teuthology.orchestra.run.smithi045.stdout:Installed: 2024-09-17T11:41:52.210 INFO:teuthology.orchestra.run.smithi045.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-17T11:41:52.210 INFO:teuthology.orchestra.run.smithi045.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-17T11:41:52.210 INFO:teuthology.orchestra.run.smithi045.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-17T11:41:52.210 INFO:teuthology.orchestra.run.smithi045.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-17T11:41:52.210 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:41:52.210 INFO:teuthology.orchestra.run.smithi045.stdout:Complete! 2024-09-17T11:41:52.295 DEBUG:teuthology.orchestra.remote:smithi165:/tmp/tmp.XFA9XxwtUp is 6KB 2024-09-17T11:41:52.343 DEBUG:teuthology.orchestra.run.smithi165:> rm -fr /tmp/tmp.XFA9XxwtUp 2024-09-17T11:41:52.358 DEBUG:teuthology.orchestra.run.smithi165:> sudo /bin/ls /boot/loader/entries || true 2024-09-17T11:41:52.421 INFO:teuthology.orchestra.run.smithi165.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-17T11:41:52.421 INFO:teuthology.orchestra.run.smithi165.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-17T11:41:52.421 INFO:teuthology.orchestra.run.smithi165.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-17T11:41:52.421 INFO:teuthology.orchestra.run.smithi165.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-17T11:41:52.423 DEBUG:teuthology.orchestra.run.smithi165:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-17T11:41:52.549 DEBUG:teuthology.orchestra.run.smithi045:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T11:41:52.582 DEBUG:teuthology.orchestra.run.smithi165:> sudo shutdown -r now 2024-09-17T11:41:52.631 INFO:teuthology.orchestra.run.smithi045.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T11:41:52.631 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T11:41:52.631 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-17T11:41:52.631 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-17T11:41:52.632 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-17T11:41:52.632 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-09-17T11:41:52.632 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi045.front.sepia.ceph.com, path=None, version=distro) 2024-09-17T11:41:52.632 DEBUG:teuthology.orchestra.run.smithi045:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T11:41:52.690 INFO:teuthology.orchestra.run.smithi045.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T11:41:52.691 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T11:41:52.691 DEBUG:teuthology.orchestra.run.smithi045:> sudo rpm -qi grub2-tools 2024-09-17T11:41:52.722 INFO:teuthology.orchestra.run.smithi045.stdout:Name : grub2-tools 2024-09-17T11:41:52.723 INFO:teuthology.orchestra.run.smithi045.stdout:Epoch : 1 2024-09-17T11:41:52.723 INFO:teuthology.orchestra.run.smithi045.stdout:Version : 2.06 2024-09-17T11:41:52.723 INFO:teuthology.orchestra.run.smithi045.stdout:Release : 82.el9 2024-09-17T11:41:52.723 INFO:teuthology.orchestra.run.smithi045.stdout:Architecture: x86_64 2024-09-17T11:41:52.723 INFO:teuthology.orchestra.run.smithi045.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-17T11:41:52.723 INFO:teuthology.orchestra.run.smithi045.stdout:Group : Unspecified 2024-09-17T11:41:52.723 INFO:teuthology.orchestra.run.smithi045.stdout:Size : 8274670 2024-09-17T11:41:52.723 INFO:teuthology.orchestra.run.smithi045.stdout:License : GPLv3+ 2024-09-17T11:41:52.723 INFO:teuthology.orchestra.run.smithi045.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-17T11:41:52.723 INFO:teuthology.orchestra.run.smithi045.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-17T11:41:52.723 INFO:teuthology.orchestra.run.smithi045.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-17T11:41:52.723 INFO:teuthology.orchestra.run.smithi045.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-17T11:41:52.723 INFO:teuthology.orchestra.run.smithi045.stdout:Packager : builder@centos.org 2024-09-17T11:41:52.724 INFO:teuthology.orchestra.run.smithi045.stdout:Vendor : CentOS 2024-09-17T11:41:52.724 INFO:teuthology.orchestra.run.smithi045.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-17T11:41:52.724 INFO:teuthology.orchestra.run.smithi045.stdout:Summary : Support tools for GRUB. 2024-09-17T11:41:52.724 INFO:teuthology.orchestra.run.smithi045.stdout:Description : 2024-09-17T11:41:52.724 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:41:52.724 INFO:teuthology.orchestra.run.smithi045.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-17T11:41:52.724 INFO:teuthology.orchestra.run.smithi045.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-17T11:41:52.724 INFO:teuthology.orchestra.run.smithi045.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-17T11:41:52.724 INFO:teuthology.orchestra.run.smithi045.stdout:hardware devices. 2024-09-17T11:41:52.724 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:41:52.724 INFO:teuthology.orchestra.run.smithi045.stdout:This subpackage provides tools for support of all platforms. 2024-09-17T11:41:52.725 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-17T11:41:52.725 INFO:teuthology.task.kernel:Updating grub on smithi045 to boot 5.14.0-508.el9.x86_64 2024-09-17T11:41:52.725 DEBUG:teuthology.orchestra.run.smithi045:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-17T11:41:53.513 INFO:teuthology.orchestra.run.smithi045.stderr:Generating grub configuration file ... 2024-09-17T11:41:54.764 INFO:teuthology.orchestra.run.smithi045.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-17T11:41:54.804 INFO:teuthology.orchestra.run.smithi045.stderr:done 2024-09-17T11:41:54.806 DEBUG:teuthology.orchestra.run.smithi045:> mktemp 2024-09-17T11:41:54.821 INFO:teuthology.orchestra.run.smithi045.stdout:/tmp/tmp.qdxxn9ab8J 2024-09-17T11:41:54.821 DEBUG:teuthology.orchestra.run.smithi045:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.qdxxn9ab8J 2024-09-17T11:41:54.888 DEBUG:teuthology.orchestra.run.smithi045:> sudo chmod 0666 /tmp/tmp.qdxxn9ab8J 2024-09-17T11:41:55.136 DEBUG:teuthology.orchestra.remote:smithi045:/tmp/tmp.qdxxn9ab8J is 6KB 2024-09-17T11:41:55.185 DEBUG:teuthology.orchestra.run.smithi045:> rm -fr /tmp/tmp.qdxxn9ab8J 2024-09-17T11:41:55.200 DEBUG:teuthology.orchestra.run.smithi045:> sudo /bin/ls /boot/loader/entries || true 2024-09-17T11:41:55.262 INFO:teuthology.orchestra.run.smithi045.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-17T11:41:55.262 INFO:teuthology.orchestra.run.smithi045.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-17T11:41:55.262 INFO:teuthology.orchestra.run.smithi045.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-17T11:41:55.262 INFO:teuthology.orchestra.run.smithi045.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-17T11:41:55.264 DEBUG:teuthology.orchestra.run.smithi045:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-17T11:41:55.428 DEBUG:teuthology.orchestra.run.smithi045:> sudo shutdown -r now 2024-09-17T11:42:22.611 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-09-17T11:42:22.612 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi165.front.sepia.ceph.com' 2024-09-17T11:42:22.613 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi165.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T11:42:25.433 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-09-17T11:42:25.433 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi045.front.sepia.ceph.com' 2024-09-17T11:42:25.434 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi045.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T11:42:40.961 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.165 2024-09-17T11:42:43.773 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.45 2024-09-17T11:42:49.968 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi165.front.sepia.ceph.com' 2024-09-17T11:42:49.968 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi165.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T11:42:52.777 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi045.front.sepia.ceph.com' 2024-09-17T11:42:52.777 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi045.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T11:42:53.025 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.165 2024-09-17T11:42:55.837 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.45 2024-09-17T11:43:05.034 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi165.front.sepia.ceph.com' 2024-09-17T11:43:05.034 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi165.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T11:43:07.841 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi045.front.sepia.ceph.com' 2024-09-17T11:43:07.841 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi045.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T11:43:23.453 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.165 2024-09-17T11:43:38.469 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi165.front.sepia.ceph.com' 2024-09-17T11:43:38.470 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi165.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T11:43:38.909 DEBUG:teuthology.orchestra.run.smithi165:> true 2024-09-17T11:43:39.244 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi165.front.sepia.ceph.com' 2024-09-17T11:43:39.244 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-508.el9.x86_64"... 2024-09-17T11:43:39.244 DEBUG:teuthology.orchestra.run.smithi165:> uname -r 2024-09-17T11:43:39.260 INFO:teuthology.orchestra.run.smithi165.stdout:5.14.0-508.el9.x86_64 2024-09-17T11:43:39.260 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-17T11:43:39.261 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-17T11:43:39.261 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-17T11:43:39.612 DEBUG:teuthology.orchestra.run.smithi045:> true 2024-09-17T11:43:40.262 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-09-17T11:43:40.262 DEBUG:teuthology.orchestra.run.smithi165:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-17T11:43:40.352 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi045.front.sepia.ceph.com' 2024-09-17T11:43:40.353 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-508.el9.x86_64"... 2024-09-17T11:43:40.353 DEBUG:teuthology.orchestra.run.smithi045:> uname -r 2024-09-17T11:43:40.390 INFO:teuthology.orchestra.run.smithi165.stdout:ttyS1 2024-09-17T11:43:40.394 INFO:teuthology.orchestra.run.smithi045.stdout:5.14.0-508.el9.x86_64 2024-09-17T11:43:40.395 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-17T11:43:40.395 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-17T11:43:40.395 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-17T11:43:40.410 DEBUG:teuthology.parallel:result is None 2024-09-17T11:43:41.395 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-09-17T11:43:41.396 DEBUG:teuthology.orchestra.run.smithi045:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-17T11:43:41.474 INFO:teuthology.orchestra.run.smithi045.stdout:ttyS1 2024-09-17T11:43:41.563 DEBUG:teuthology.parallel:result is None 2024-09-17T11:43:41.563 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-17T11:43:41.570 INFO:teuthology.task.internal:Creating test directory... 2024-09-17T11:43:41.571 DEBUG:teuthology.orchestra.run.smithi045:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-17T11:43:41.573 DEBUG:teuthology.orchestra.run.smithi165:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-17T11:43:41.588 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-17T11:43:41.662 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-17T11:43:41.673 INFO:teuthology.task.internal:Creating archive directory... 2024-09-17T11:43:41.673 DEBUG:teuthology.orchestra.run.smithi045:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-17T11:43:41.675 DEBUG:teuthology.orchestra.run.smithi165:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-17T11:43:41.710 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-17T11:43:41.718 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-17T11:43:41.719 DEBUG:teuthology.orchestra.run.smithi045:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-17T11:43:41.746 DEBUG:teuthology.orchestra.run.smithi165:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-17T11:43:41.778 INFO:teuthology.orchestra.run.smithi045.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T11:43:41.788 INFO:teuthology.orchestra.run.smithi045.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T11:43:42.113 INFO:teuthology.orchestra.run.smithi165.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T11:43:42.125 INFO:teuthology.orchestra.run.smithi165.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T11:43:42.127 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-17T11:43:42.133 INFO:teuthology.task.internal:Configuring sudo... 2024-09-17T11:43:42.133 DEBUG:teuthology.orchestra.run.smithi045:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-17T11:43:42.136 DEBUG:teuthology.orchestra.run.smithi165:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-17T11:43:42.165 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-17T11:43:42.173 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-17T11:43:42.174 DEBUG:teuthology.orchestra.run.smithi045:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-17T11:43:42.205 DEBUG:teuthology.orchestra.run.smithi165:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-17T11:43:42.255 DEBUG:teuthology.orchestra.run.smithi045:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T11:43:42.308 DEBUG:teuthology.orchestra.run.smithi045:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T11:43:42.357 DEBUG:teuthology.orchestra.run.smithi045:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T11:43:42.443 DEBUG:teuthology.orchestra.run.smithi045:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T11:43:42.487 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2024-09-17T11:43:42.487 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-17T11:43:42.556 DEBUG:teuthology.orchestra.run.smithi165:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T11:43:42.619 DEBUG:teuthology.orchestra.run.smithi165:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T11:43:42.703 DEBUG:teuthology.orchestra.run.smithi165:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T11:43:42.765 DEBUG:teuthology.orchestra.run.smithi165:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T11:43:42.842 DEBUG:teuthology.orchestra.run.smithi165:> set -ex 2024-09-17T11:43:42.843 DEBUG:teuthology.orchestra.run.smithi165:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-17T11:43:42.913 DEBUG:teuthology.orchestra.run.smithi045:> sudo service rsyslog restart 2024-09-17T11:43:42.915 DEBUG:teuthology.orchestra.run.smithi165:> sudo service rsyslog restart 2024-09-17T11:43:42.964 INFO:teuthology.orchestra.run.smithi045.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T11:43:43.049 INFO:teuthology.orchestra.run.smithi165.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T11:43:43.472 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-17T11:43:43.480 INFO:teuthology.task.internal:Starting timer... 2024-09-17T11:43:43.480 INFO:teuthology.run_tasks:Running task pcp... 2024-09-17T11:43:43.500 INFO:teuthology.run_tasks:Running task selinux... 2024-09-17T11:43:43.508 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-17T11:43:43.509 DEBUG:teuthology.orchestra.run.smithi045:> sudo service auditd rotate 2024-09-17T11:43:43.567 INFO:teuthology.orchestra.run.smithi045.stdout:Rotating logs: 2024-09-17T11:43:43.569 DEBUG:teuthology.orchestra.run.smithi165:> sudo service auditd rotate 2024-09-17T11:43:43.714 INFO:teuthology.orchestra.run.smithi165.stdout:Rotating logs: 2024-09-17T11:43:43.716 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-17T11:43:43.716 DEBUG:teuthology.orchestra.run.smithi045:> /usr/sbin/getenforce 2024-09-17T11:43:43.741 INFO:teuthology.orchestra.run.smithi045.stdout:Permissive 2024-09-17T11:43:43.742 DEBUG:teuthology.orchestra.run.smithi165:> /usr/sbin/getenforce 2024-09-17T11:43:43.852 INFO:teuthology.orchestra.run.smithi165.stdout:Permissive 2024-09-17T11:43:43.852 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi045.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi165.front.sepia.ceph.com': 'permissive'} 2024-09-17T11:43:43.852 DEBUG:teuthology.orchestra.run.smithi045:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-17T11:43:43.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T11:43:43.881 DEBUG:teuthology.orchestra.run.smithi165:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-17T11:43:43.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T11:43:43.907 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-17T11:43:43.907 DEBUG:teuthology.orchestra.run.smithi045:> sudo /usr/sbin/setenforce permissive 2024-09-17T11:43:43.955 DEBUG:teuthology.orchestra.run.smithi165:> sudo /usr/sbin/setenforce permissive 2024-09-17T11:43:43.990 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-17T11:43:44.000 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-17T11:43:44.011 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-17T11:43:44.012 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi045.front.sepia.ceph.com,smithi165.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-17T11:47:34.019 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi045.front.sepia.ceph.com'), Remote(name='ubuntu@smithi165.front.sepia.ceph.com')] 2024-09-17T11:47:34.021 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi045.front.sepia.ceph.com' 2024-09-17T11:47:34.022 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi045.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T11:47:34.103 DEBUG:teuthology.orchestra.run.smithi045:> true 2024-09-17T11:47:34.185 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi045.front.sepia.ceph.com' 2024-09-17T11:47:34.186 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi165.front.sepia.ceph.com' 2024-09-17T11:47:34.186 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi165.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T11:47:34.262 DEBUG:teuthology.orchestra.run.smithi165:> true 2024-09-17T11:47:34.340 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi165.front.sepia.ceph.com' 2024-09-17T11:47:34.340 INFO:teuthology.run_tasks:Running task clock... 2024-09-17T11:47:34.349 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-17T11:47:34.349 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-17T11:47:34.350 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T11:47:34.352 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-17T11:47:34.352 DEBUG:teuthology.orchestra.run.smithi165:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T11:47:34.382 INFO:teuthology.orchestra.run.smithi045.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-17T11:47:34.398 INFO:teuthology.orchestra.run.smithi045.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-17T11:47:34.410 INFO:teuthology.orchestra.run.smithi165.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-17T11:47:34.424 INFO:teuthology.orchestra.run.smithi045.stderr:sudo: ntpd: command not found 2024-09-17T11:47:34.425 INFO:teuthology.orchestra.run.smithi165.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-17T11:47:34.436 INFO:teuthology.orchestra.run.smithi045.stdout:506 Cannot talk to daemon 2024-09-17T11:47:34.451 INFO:teuthology.orchestra.run.smithi165.stderr:sudo: ntpd: command not found 2024-09-17T11:47:34.451 INFO:teuthology.orchestra.run.smithi045.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-17T11:47:34.462 INFO:teuthology.orchestra.run.smithi165.stdout:506 Cannot talk to daemon 2024-09-17T11:47:34.466 INFO:teuthology.orchestra.run.smithi045.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-17T11:47:34.476 INFO:teuthology.orchestra.run.smithi165.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-17T11:47:34.490 INFO:teuthology.orchestra.run.smithi165.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-17T11:47:34.533 INFO:teuthology.orchestra.run.smithi045.stderr:bash: line 1: ntpq: command not found 2024-09-17T11:47:34.536 INFO:teuthology.orchestra.run.smithi045.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T11:47:34.536 INFO:teuthology.orchestra.run.smithi045.stdout:=============================================================================== 2024-09-17T11:47:34.536 INFO:teuthology.orchestra.run.smithi045.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T11:47:34.536 INFO:teuthology.orchestra.run.smithi045.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T11:47:34.536 INFO:teuthology.orchestra.run.smithi045.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T11:47:34.537 INFO:teuthology.orchestra.run.smithi045.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T11:47:34.550 INFO:teuthology.orchestra.run.smithi165.stderr:bash: line 1: ntpq: command not found 2024-09-17T11:47:34.553 INFO:teuthology.orchestra.run.smithi165.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T11:47:34.553 INFO:teuthology.orchestra.run.smithi165.stdout:=============================================================================== 2024-09-17T11:47:34.553 INFO:teuthology.orchestra.run.smithi165.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T11:47:34.553 INFO:teuthology.orchestra.run.smithi165.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T11:47:34.553 INFO:teuthology.orchestra.run.smithi165.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T11:47:34.553 INFO:teuthology.orchestra.run.smithi165.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T11:47:34.554 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-17T11:47:34.650 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_REFRESH_FAILED'], 'log-only-match': ['CEPHADM_'], 'sha1': '7498493279982dfe87f00616198a5967475f6169'} 2024-09-17T11:47:34.651 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-17T11:47:34.651 INFO:tasks.cephadm:Cluster fsid is a12537de-74ea-11ef-bceb-c7b262605968 2024-09-17T11:47:34.651 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-17T11:47:34.651 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-09-17T11:47:34.652 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi045': '172.21.15.45', 'mon.smithi165': '172.21.15.165'} 2024-09-17T11:47:34.652 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-17T11:47:34.652 DEBUG:teuthology.orchestra.run.smithi045:> sudo hostname $(hostname -s) 2024-09-17T11:47:34.679 DEBUG:teuthology.orchestra.run.smithi165:> sudo hostname $(hostname -s) 2024-09-17T11:47:34.705 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-17T11:47:34.705 DEBUG:teuthology.orchestra.run.smithi045:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-17T11:47:34.971 INFO:teuthology.orchestra.run.smithi045.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 17 11:47 /home/ubuntu/cephtest/cephadm 2024-09-17T11:47:34.971 DEBUG:teuthology.orchestra.run.smithi165:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-17T11:47:35.107 INFO:teuthology.orchestra.run.smithi165.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 17 11:47 /home/ubuntu/cephtest/cephadm 2024-09-17T11:47:35.108 DEBUG:teuthology.orchestra.run.smithi045:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-17T11:47:35.125 DEBUG:teuthology.orchestra.run.smithi165:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-17T11:47:35.149 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-17T11:47:35.149 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-17T11:47:35.168 DEBUG:teuthology.orchestra.run.smithi165:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-17T11:47:36.674 INFO:teuthology.orchestra.run.smithi165.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-17T11:47:36.733 INFO:teuthology.orchestra.run.smithi045.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-17T11:47:49.518 INFO:teuthology.orchestra.run.smithi165.stdout:{ 2024-09-17T11:47:49.518 INFO:teuthology.orchestra.run.smithi165.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-17T11:47:49.519 INFO:teuthology.orchestra.run.smithi165.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-17T11:47:49.519 INFO:teuthology.orchestra.run.smithi165.stdout: "repo_digests": [ 2024-09-17T11:47:49.519 INFO:teuthology.orchestra.run.smithi165.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-17T11:47:49.519 INFO:teuthology.orchestra.run.smithi165.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-17T11:47:49.519 INFO:teuthology.orchestra.run.smithi165.stdout: ] 2024-09-17T11:47:49.519 INFO:teuthology.orchestra.run.smithi165.stdout:} 2024-09-17T11:47:54.097 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T11:47:54.097 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-17T11:47:54.097 INFO:teuthology.orchestra.run.smithi045.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-17T11:47:54.097 INFO:teuthology.orchestra.run.smithi045.stdout: "repo_digests": [ 2024-09-17T11:47:54.097 INFO:teuthology.orchestra.run.smithi045.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-17T11:47:54.098 INFO:teuthology.orchestra.run.smithi045.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-17T11:47:54.098 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-17T11:47:54.098 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T11:47:54.116 DEBUG:teuthology.orchestra.run.smithi045:> sudo mkdir -p /etc/ceph 2024-09-17T11:47:54.165 DEBUG:teuthology.orchestra.run.smithi165:> sudo mkdir -p /etc/ceph 2024-09-17T11:47:54.198 DEBUG:teuthology.orchestra.run.smithi045:> sudo chmod 777 /etc/ceph 2024-09-17T11:47:54.223 DEBUG:teuthology.orchestra.run.smithi165:> sudo chmod 777 /etc/ceph 2024-09-17T11:47:54.266 INFO:tasks.cephadm:Writing seed config... 2024-09-17T11:47:54.267 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-17T11:47:54.267 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-17T11:47:54.267 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-17T11:47:54.267 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-17T11:47:54.267 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-17T11:47:54.267 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-17T11:47:54.267 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-17T11:47:54.268 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-17T11:47:54.268 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2024-09-17T11:47:54.268 DEBUG:teuthology.orchestra.run.smithi045:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-17T11:47:54.283 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = a12537de-74ea-11ef-bceb-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-17T11:47:54.284 DEBUG:teuthology.orchestra.run.smithi045:mon.smithi045> sudo journalctl -f -n 0 -u ceph-a12537de-74ea-11ef-bceb-c7b262605968@mon.smithi045.service 2024-09-17T11:47:54.326 INFO:tasks.cephadm:Bootstrapping... 2024-09-17T11:47:54.326 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid a12537de-74ea-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.45 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-17T11:47:54.523 INFO:teuthology.orchestra.run.smithi045.stderr:-------------------------------------------------------------------------------- 2024-09-17T11:47:54.524 INFO:teuthology.orchestra.run.smithi045.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', 'a12537de-74ea-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.45', '--skip-admin-label'] 2024-09-17T11:47:54.546 INFO:teuthology.orchestra.run.smithi045.stderr:/bin/podman: 5.2.2 2024-09-17T11:47:54.549 INFO:teuthology.orchestra.run.smithi045.stderr:Verifying podman|docker is present... 2024-09-17T11:47:54.570 INFO:teuthology.orchestra.run.smithi045.stderr:/bin/podman: 5.2.2 2024-09-17T11:47:54.572 INFO:teuthology.orchestra.run.smithi045.stderr:Verifying lvm2 is present... 2024-09-17T11:47:54.572 INFO:teuthology.orchestra.run.smithi045.stderr:Verifying time synchronization is in place... 2024-09-17T11:47:54.579 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-17T11:47:54.586 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl: inactive 2024-09-17T11:47:54.593 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl: enabled 2024-09-17T11:47:54.600 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl: active 2024-09-17T11:47:54.600 INFO:teuthology.orchestra.run.smithi045.stderr:Unit chronyd.service is enabled and running 2024-09-17T11:47:54.600 INFO:teuthology.orchestra.run.smithi045.stderr:Repeating the final host check... 2024-09-17T11:47:54.627 INFO:teuthology.orchestra.run.smithi045.stderr:/bin/podman: 5.2.2 2024-09-17T11:47:54.629 INFO:teuthology.orchestra.run.smithi045.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-17T11:47:54.629 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl is present 2024-09-17T11:47:54.629 INFO:teuthology.orchestra.run.smithi045.stderr:lvcreate is present 2024-09-17T11:47:54.635 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-17T11:47:54.642 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl: inactive 2024-09-17T11:47:54.649 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl: enabled 2024-09-17T11:47:54.656 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl: active 2024-09-17T11:47:54.656 INFO:teuthology.orchestra.run.smithi045.stderr:Unit chronyd.service is enabled and running 2024-09-17T11:47:54.657 INFO:teuthology.orchestra.run.smithi045.stderr:Host looks OK 2024-09-17T11:47:54.657 INFO:teuthology.orchestra.run.smithi045.stderr:Cluster fsid: a12537de-74ea-11ef-bceb-c7b262605968 2024-09-17T11:47:54.657 INFO:teuthology.orchestra.run.smithi045.stderr:Acquiring lock 140408923613072 on /run/cephadm/a12537de-74ea-11ef-bceb-c7b262605968.lock 2024-09-17T11:47:54.657 INFO:teuthology.orchestra.run.smithi045.stderr:Lock 140408923613072 acquired on /run/cephadm/a12537de-74ea-11ef-bceb-c7b262605968.lock 2024-09-17T11:47:54.658 INFO:teuthology.orchestra.run.smithi045.stderr:Verifying IP 172.21.15.45 port 3300 ... 2024-09-17T11:47:54.658 INFO:teuthology.orchestra.run.smithi045.stderr:Verifying IP 172.21.15.45 port 6789 ... 2024-09-17T11:47:54.658 INFO:teuthology.orchestra.run.smithi045.stderr:Base mon IP is 172.21.15.45, final addrv is [v2:172.21.15.45:3300,v1:172.21.15.45:6789] 2024-09-17T11:47:54.660 INFO:teuthology.orchestra.run.smithi045.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.45 metric 100 2024-09-17T11:47:54.660 INFO:teuthology.orchestra.run.smithi045.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.45 metric 100 2024-09-17T11:47:54.663 INFO:teuthology.orchestra.run.smithi045.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-17T11:47:54.663 INFO:teuthology.orchestra.run.smithi045.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2024-09-17T11:47:54.663 INFO:teuthology.orchestra.run.smithi045.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1794sec hoplimit 64 pref medium 2024-09-17T11:47:54.666 INFO:teuthology.orchestra.run.smithi045.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-17T11:47:54.666 INFO:teuthology.orchestra.run.smithi045.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-17T11:47:54.666 INFO:teuthology.orchestra.run.smithi045.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-17T11:47:54.666 INFO:teuthology.orchestra.run.smithi045.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-09-17T11:47:54.666 INFO:teuthology.orchestra.run.smithi045.stderr:/sbin/ip: inet6 fe80::ec4:7aff:febd:12be/64 scope link 2024-09-17T11:47:54.666 INFO:teuthology.orchestra.run.smithi045.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-17T11:47:54.667 INFO:teuthology.orchestra.run.smithi045.stderr:Mon IP `172.21.15.45` is in CIDR network `172.21.0.0/20` 2024-09-17T11:47:54.667 INFO:teuthology.orchestra.run.smithi045.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-17T11:47:54.667 INFO:teuthology.orchestra.run.smithi045.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-17T11:47:54.772 INFO:teuthology.orchestra.run.smithi045.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-17T11:47:55.154 INFO:teuthology.orchestra.run.smithi045.stderr:/bin/podman: Getting image source signatures 2024-09-17T11:47:55.155 INFO:teuthology.orchestra.run.smithi045.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-17T11:47:55.155 INFO:teuthology.orchestra.run.smithi045.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-17T11:47:55.155 INFO:teuthology.orchestra.run.smithi045.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-17T11:47:55.155 INFO:teuthology.orchestra.run.smithi045.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-17T11:47:55.155 INFO:teuthology.orchestra.run.smithi045.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-17T11:47:55.155 INFO:teuthology.orchestra.run.smithi045.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-17T11:47:55.325 INFO:teuthology.orchestra.run.smithi045.stderr:/bin/podman: Writing manifest to image destination 2024-09-17T11:47:55.478 INFO:teuthology.orchestra.run.smithi045.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-17T11:47:55.884 INFO:teuthology.orchestra.run.smithi045.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-17T11:47:56.436 INFO:teuthology.orchestra.run.smithi045.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-17T11:47:56.436 INFO:teuthology.orchestra.run.smithi045.stderr:Extracting ceph user uid/gid from container image... 2024-09-17T11:47:56.869 INFO:teuthology.orchestra.run.smithi045.stderr:stat: 167 167 2024-09-17T11:47:57.342 INFO:teuthology.orchestra.run.smithi045.stderr:Creating initial keys... 2024-09-17T11:47:57.717 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph-authtool: AQBtbOlmWLRGJhAAAkoEK3urlzzmPk5Yvq8uUg== 2024-09-17T11:47:58.631 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph-authtool: AQBubOlmAdUkHxAAdVUuq5trjpzvBVI4gSyxzw== 2024-09-17T11:47:59.521 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph-authtool: AQBvbOlm4G6aGBAAFVfwr7TpjFqerBynzifu2Q== 2024-09-17T11:48:00.103 INFO:teuthology.orchestra.run.smithi045.stderr:Creating initial monmap... 2024-09-17T11:48:00.494 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-17T11:48:00.494 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-17T11:48:00.494 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to a12537de-74ea-11ef-bceb-c7b262605968 2024-09-17T11:48:00.494 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-17T11:48:00.984 INFO:teuthology.orchestra.run.smithi045.stderr:monmaptool for smithi045 [v2:172.21.15.45:3300,v1:172.21.15.45:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-17T11:48:00.984 INFO:teuthology.orchestra.run.smithi045.stderr:setting min_mon_release = octopus 2024-09-17T11:48:00.984 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/monmaptool: set fsid to a12537de-74ea-11ef-bceb-c7b262605968 2024-09-17T11:48:00.984 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-17T11:48:00.984 INFO:teuthology.orchestra.run.smithi045.stderr: 2024-09-17T11:48:00.984 INFO:teuthology.orchestra.run.smithi045.stderr:Creating mon... 2024-09-17T11:48:01.824 INFO:teuthology.orchestra.run.smithi045.stderr:create mon.smithi045 on 2024-09-17T11:48:02.040 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-17T11:48:02.200 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-a12537de-74ea-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-a12537de-74ea-11ef-bceb-c7b262605968.target. 2024-09-17T11:48:02.200 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-a12537de-74ea-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-a12537de-74ea-11ef-bceb-c7b262605968.target. 2024-09-17T11:48:02.508 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl: Failed to reset failed state of unit ceph-a12537de-74ea-11ef-bceb-c7b262605968@mon.smithi045.service: Unit ceph-a12537de-74ea-11ef-bceb-c7b262605968@mon.smithi045.service not loaded. 2024-09-17T11:48:02.516 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl: Created symlink /etc/systemd/system/ceph-a12537de-74ea-11ef-bceb-c7b262605968.target.wants/ceph-a12537de-74ea-11ef-bceb-c7b262605968@mon.smithi045.service → /etc/systemd/system/ceph-a12537de-74ea-11ef-bceb-c7b262605968@.service. 2024-09-17T11:48:03.169 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:03 smithi045 podman[23378]: 2024-09-17 11:48:03.129590017 +0000 UTC m=+0.329922684 container start ac8e46e7e27c360a23110fa86b12db48a54fdfb09c5e305daf4d84dc47c28701 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, ceph=True, release=754, architecture=x86_64, vcs-type=git, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, io.openshift.expose-services=, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD) 2024-09-17T11:48:03.169 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:03 smithi045 ceph-mon[23398]: mkfs a12537de-74ea-11ef-bceb-c7b262605968 2024-09-17T11:48:03.223 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl: disabled 2024-09-17T11:48:03.230 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl: inactive 2024-09-17T11:48:03.231 INFO:teuthology.orchestra.run.smithi045.stderr:firewalld.service is not enabled 2024-09-17T11:48:03.231 INFO:teuthology.orchestra.run.smithi045.stderr:Not possible to enable service . firewalld.service is not available 2024-09-17T11:48:03.232 INFO:teuthology.orchestra.run.smithi045.stderr:Waiting for mon to start... 2024-09-17T11:48:03.232 INFO:teuthology.orchestra.run.smithi045.stderr:Waiting for mon... 2024-09-17T11:48:03.455 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:03 smithi045 ceph-mon[23398]: mon.smithi045 is new leader, mons smithi045 in quorum (ranks 0) 2024-09-17T11:48:03.455 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:03 smithi045 bash[23378]: ac8e46e7e27c360a23110fa86b12db48a54fdfb09c5e305daf4d84dc47c28701 2024-09-17T11:48:03.455 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:03 smithi045 systemd[1]: Started Ceph mon.smithi045 for a12537de-74ea-11ef-bceb-c7b262605968. 2024-09-17T11:48:03.753 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: cluster: 2024-09-17T11:48:03.753 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: id: a12537de-74ea-11ef-bceb-c7b262605968 2024-09-17T11:48:03.753 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-17T11:48:03.753 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: 2024-09-17T11:48:03.753 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: services: 2024-09-17T11:48:03.753 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi045 (age 0.584744s) 2024-09-17T11:48:03.753 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-17T11:48:03.754 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-17T11:48:03.754 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: 2024-09-17T11:48:03.754 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: data: 2024-09-17T11:48:03.754 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-17T11:48:03.754 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-17T11:48:03.754 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-17T11:48:03.754 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: pgs: 2024-09-17T11:48:03.754 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: 2024-09-17T11:48:04.301 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:04 smithi045 ceph-mon[23398]: mon.smithi045 is new leader, mons smithi045 in quorum (ranks 0) 2024-09-17T11:48:04.301 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:04 smithi045 ceph-mon[23398]: monmap e1: 1 mons at {smithi045=[v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0]} 2024-09-17T11:48:04.301 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:04 smithi045 ceph-mon[23398]: fsmap 2024-09-17T11:48:04.301 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:04 smithi045 ceph-mon[23398]: osdmap e1: 0 total, 0 up, 0 in 2024-09-17T11:48:04.301 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:04 smithi045 ceph-mon[23398]: mgrmap e1: no daemons active 2024-09-17T11:48:04.301 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:04 smithi045 ceph-mon[23398]: from='client.? 172.21.15.45:0/1649258793' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T11:48:04.303 INFO:teuthology.orchestra.run.smithi045.stderr:mon is available 2024-09-17T11:48:04.303 INFO:teuthology.orchestra.run.smithi045.stderr:Assimilating anything we can from ceph.conf... 2024-09-17T11:48:04.826 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: 2024-09-17T11:48:04.826 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: [global] 2024-09-17T11:48:04.827 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: fsid = a12537de-74ea-11ef-bceb-c7b262605968 2024-09-17T11:48:04.827 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.45:3300,v1:172.21.15.45:6789] 2024-09-17T11:48:04.827 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-17T11:48:04.827 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-17T11:48:04.827 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-17T11:48:04.827 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-17T11:48:04.827 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: 2024-09-17T11:48:04.827 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: [mgr] 2024-09-17T11:48:04.827 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-17T11:48:04.827 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: 2024-09-17T11:48:04.827 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: [osd] 2024-09-17T11:48:04.827 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-17T11:48:04.828 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-17T11:48:05.416 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:05 smithi045 ceph-mon[23398]: from='client.? 172.21.15.45:0/3600413751' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-17T11:48:05.416 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:05 smithi045 ceph-mon[23398]: from='client.? 172.21.15.45:0/3600413751' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-17T11:48:05.418 INFO:teuthology.orchestra.run.smithi045.stderr:Generating new minimal ceph.conf... 2024-09-17T11:48:06.424 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:06 smithi045 ceph-mon[23398]: from='client.? 172.21.15.45:0/2769035626' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:48:06.425 INFO:teuthology.orchestra.run.smithi045.stderr:Restarting the monitor... 2024-09-17T11:48:06.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:06 smithi045 systemd[1]: Stopping Ceph mon.smithi045 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T11:48:06.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:06 smithi045 bash[23616]: Error: no container with name or ID "ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon.smithi045" found: no such container 2024-09-17T11:48:06.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:06 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045[23394]: 2024-09-17T11:48:06.574+0000 7f62f05f2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi045 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T11:48:06.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:06 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045[23394]: 2024-09-17T11:48:06.574+0000 7f62f05f2700 -1 mon.smithi045@0(leader) e1 *** Got Signal Terminated *** 2024-09-17T11:48:06.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:06 smithi045 podman[23627]: 2024-09-17 11:48:06.605834593 +0000 UTC m=+0.120306402 container died ac8e46e7e27c360a23110fa86b12db48a54fdfb09c5e305daf4d84dc47c28701 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., vcs-type=git, release=754, ceph=True, com.redhat.component=centos-stream-container, GIT_CLEAN=True, version=8, distribution-scope=public, io.openshift.tags=base centos centos-stream) 2024-09-17T11:48:07.210 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:06 smithi045 podman[23627]: 2024-09-17 11:48:06.957764816 +0000 UTC m=+0.472236625 container cleanup ac8e46e7e27c360a23110fa86b12db48a54fdfb09c5e305daf4d84dc47c28701 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, vcs-type=git, com.redhat.component=centos-stream-container, architecture=x86_64, release=754, RELEASE=HEAD, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, vendor=Red Hat, Inc., distribution-scope=public, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, ceph=True, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-17T11:48:07.210 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:06 smithi045 bash[23627]: ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045 2024-09-17T11:48:07.210 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:06 smithi045 bash[23655]: Error: no container with name or ID "ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon.smithi045" found: no such container 2024-09-17T11:48:07.516 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 podman[23639]: 2024-09-17 11:48:07.208897517 +0000 UTC m=+0.603502187 container remove ac8e46e7e27c360a23110fa86b12db48a54fdfb09c5e305daf4d84dc47c28701 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045, RELEASE=HEAD, ceph=True, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, io.openshift.expose-services=, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, release=754, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64) 2024-09-17T11:48:07.516 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 systemd[1]: ceph-a12537de-74ea-11ef-bceb-c7b262605968@mon.smithi045.service: Deactivated successfully. 2024-09-17T11:48:07.516 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 systemd[1]: Stopped Ceph mon.smithi045 for a12537de-74ea-11ef-bceb-c7b262605968. 2024-09-17T11:48:07.516 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 systemd[1]: Starting Ceph mon.smithi045 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T11:48:07.516 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 podman[23718]: 2024-09-17 11:48:07.42649185 +0000 UTC m=+0.096203330 container create c4256bdf51b7d955c4309544040231c06bc9b4544fcc6131c1d2d633fb0ddccc (image=quay.io/ceph/ceph:v17.2.0, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, architecture=x86_64, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, version=8, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-17T11:48:07.517 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 podman[23718]: 2024-09-17 11:48:07.347298354 +0000 UTC m=+0.017009782 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-17T11:48:07.659 INFO:teuthology.orchestra.run.smithi045.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-17T11:48:07.876 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 podman[23718]: 2024-09-17 11:48:07.595828329 +0000 UTC m=+0.265539747 container init c4256bdf51b7d955c4309544040231c06bc9b4544fcc6131c1d2d633fb0ddccc (image=quay.io/ceph/ceph:v17.2.0, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, com.redhat.component=centos-stream-container, vcs-type=git, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, io.openshift.tags=base centos centos-stream, release=754, build-date=2022-05-03T08:36:31.336870, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, vendor=Red Hat, Inc., io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0) 2024-09-17T11:48:07.876 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 podman[23718]: 2024-09-17 11:48:07.599076673 +0000 UTC m=+0.268788091 container start c4256bdf51b7d955c4309544040231c06bc9b4544fcc6131c1d2d633fb0ddccc (image=quay.io/ceph/ceph:v17.2.0, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, maintainer=Guillaume Abrioux , name=centos-stream, release=754, RELEASE=HEAD, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., distribution-scope=public, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True) 2024-09-17T11:48:07.876 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: set uid:gid to 167:167 (ceph:ceph) 2024-09-17T11:48:07.876 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-17T11:48:07.876 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: pidfile_write: ignore empty --pid-file 2024-09-17T11:48:07.876 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: load: jerasure load: lrc 2024-09-17T11:48:07.877 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: RocksDB version: 6.15.5 2024-09-17T11:48:07.877 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-17T11:48:07.877 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Compile date Apr 18 2022 2024-09-17T11:48:07.877 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: DB SUMMARY 2024-09-17T11:48:07.877 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: DB Session ID: R3YOA8FQ8A9917HLC6KW 2024-09-17T11:48:07.877 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: CURRENT file: CURRENT 2024-09-17T11:48:07.877 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: IDENTITY file: IDENTITY 2024-09-17T11:48:07.877 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-17T11:48:07.878 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi045/store.db dir, Total Num: 1, files: 000008.sst 2024-09-17T11:48:07.878 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi045/store.db: 000010.log size: 82715 ; 2024-09-17T11:48:07.878 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.error_if_exists: 0 2024-09-17T11:48:07.878 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.create_if_missing: 0 2024-09-17T11:48:07.878 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.paranoid_checks: 1 2024-09-17T11:48:07.878 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-17T11:48:07.878 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.env: 0x560e44ac6860 2024-09-17T11:48:07.878 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.fs: Posix File System 2024-09-17T11:48:07.878 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.info_log: 0x560e45d79fa0 2024-09-17T11:48:07.878 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_file_opening_threads: 16 2024-09-17T11:48:07.878 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.statistics: (nil) 2024-09-17T11:48:07.878 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.use_fsync: 0 2024-09-17T11:48:07.878 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_log_file_size: 0 2024-09-17T11:48:07.879 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T11:48:07.879 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T11:48:07.879 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.keep_log_file_num: 1000 2024-09-17T11:48:07.879 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.recycle_log_file_num: 0 2024-09-17T11:48:07.879 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.allow_fallocate: 1 2024-09-17T11:48:07.879 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.allow_mmap_reads: 0 2024-09-17T11:48:07.879 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.allow_mmap_writes: 0 2024-09-17T11:48:07.879 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.use_direct_reads: 0 2024-09-17T11:48:07.879 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T11:48:07.879 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.create_missing_column_families: 0 2024-09-17T11:48:07.879 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.db_log_dir: 2024-09-17T11:48:07.879 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi045/store.db 2024-09-17T11:48:07.879 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T11:48:07.879 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T11:48:07.880 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T11:48:07.880 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T11:48:07.880 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T11:48:07.880 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T11:48:07.880 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.advise_random_on_open: 1 2024-09-17T11:48:07.881 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.db_write_buffer_size: 0 2024-09-17T11:48:07.881 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.write_buffer_manager: 0x560e45e6a330 2024-09-17T11:48:07.881 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T11:48:07.881 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-17T11:48:07.881 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T11:48:07.881 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T11:48:07.881 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.rate_limiter: (nil) 2024-09-17T11:48:07.881 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T11:48:07.881 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.wal_recovery_mode: 2 2024-09-17T11:48:07.881 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.enable_thread_tracking: 0 2024-09-17T11:48:07.881 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.enable_pipelined_write: 0 2024-09-17T11:48:07.881 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.unordered_write: 0 2024-09-17T11:48:07.882 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T11:48:07.882 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T11:48:07.882 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T11:48:07.882 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T11:48:07.882 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.row_cache: None 2024-09-17T11:48:07.882 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.wal_filter: None 2024-09-17T11:48:07.882 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T11:48:07.882 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.allow_ingest_behind: 0 2024-09-17T11:48:07.882 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.preserve_deletes: 0 2024-09-17T11:48:07.882 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.two_write_queues: 0 2024-09-17T11:48:07.882 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.manual_wal_flush: 0 2024-09-17T11:48:07.882 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.atomic_flush: 0 2024-09-17T11:48:07.882 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T11:48:07.883 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T11:48:07.883 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T11:48:07.883 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.log_readahead_size: 0 2024-09-17T11:48:07.883 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-17T11:48:07.883 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.best_efforts_recovery: 0 2024-09-17T11:48:07.883 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-17T11:48:07.883 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-17T11:48:07.883 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.allow_data_in_errors: 0 2024-09-17T11:48:07.883 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.db_host_id: __hostname__ 2024-09-17T11:48:07.883 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_background_jobs: 2 2024-09-17T11:48:07.883 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_background_compactions: -1 2024-09-17T11:48:07.883 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_subcompactions: 1 2024-09-17T11:48:07.883 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T11:48:07.884 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T11:48:07.884 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T11:48:07.884 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_total_wal_size: 0 2024-09-17T11:48:07.884 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T11:48:07.884 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T11:48:07.884 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T11:48:07.884 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T11:48:07.884 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_open_files: -1 2024-09-17T11:48:07.884 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.bytes_per_sync: 0 2024-09-17T11:48:07.884 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T11:48:07.884 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T11:48:07.884 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.compaction_readahead_size: 0 2024-09-17T11:48:07.884 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_background_flushes: -1 2024-09-17T11:48:07.885 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Compression algorithms supported: 2024-09-17T11:48:07.885 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T11:48:07.885 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: kZSTD supported: 0 2024-09-17T11:48:07.885 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: kXpressCompression supported: 0 2024-09-17T11:48:07.885 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: kLZ4HCCompression supported: 1 2024-09-17T11:48:07.885 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: kLZ4Compression supported: 1 2024-09-17T11:48:07.885 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: kBZip2Compression supported: 0 2024-09-17T11:48:07.885 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: kZlibCompression supported: 1 2024-09-17T11:48:07.885 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: kSnappyCompression supported: 1 2024-09-17T11:48:07.885 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T11:48:07.885 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi045/store.db/MANIFEST-000009 2024-09-17T11:48:07.885 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-17T11:48:07.885 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T11:48:07.885 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.merge_operator: 2024-09-17T11:48:07.886 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.compaction_filter: None 2024-09-17T11:48:07.886 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.compaction_filter_factory: None 2024-09-17T11:48:07.886 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.sst_partitioner_factory: None 2024-09-17T11:48:07.886 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T11:48:07.886 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T11:48:07.886 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560e45d45d00) 2024-09-17T11:48:07.886 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: cache_index_and_filter_blocks: 1 2024-09-17T11:48:07.886 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T11:48:07.886 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T11:48:07.886 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: pin_top_level_index_and_filter: 1 2024-09-17T11:48:07.886 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: index_type: 0 2024-09-17T11:48:07.886 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: data_block_index_type: 0 2024-09-17T11:48:07.886 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: index_shortening: 1 2024-09-17T11:48:07.887 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-17T11:48:07.887 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: hash_index_allow_collision: 1 2024-09-17T11:48:07.887 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: checksum: 1 2024-09-17T11:48:07.887 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: no_block_cache: 0 2024-09-17T11:48:07.887 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: block_cache: 0x560e45db0170 2024-09-17T11:48:07.887 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: block_cache_name: BinnedLRUCache 2024-09-17T11:48:07.887 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: block_cache_options: 2024-09-17T11:48:07.887 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: capacity : 536870912 2024-09-17T11:48:07.887 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: num_shard_bits : 4 2024-09-17T11:48:07.887 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: strict_capacity_limit : 0 2024-09-17T11:48:07.887 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: high_pri_pool_ratio: 0.000 2024-09-17T11:48:07.887 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: block_cache_compressed: (nil) 2024-09-17T11:48:07.887 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: persistent_cache: (nil) 2024-09-17T11:48:07.887 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: block_size: 4096 2024-09-17T11:48:07.888 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: block_size_deviation: 10 2024-09-17T11:48:07.888 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: block_restart_interval: 16 2024-09-17T11:48:07.888 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: index_block_restart_interval: 1 2024-09-17T11:48:07.888 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: metadata_block_size: 4096 2024-09-17T11:48:07.888 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: partition_filters: 0 2024-09-17T11:48:07.888 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: use_delta_encoding: 1 2024-09-17T11:48:07.888 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-17T11:48:07.888 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: whole_key_filtering: 1 2024-09-17T11:48:07.888 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: verify_compression: 0 2024-09-17T11:48:07.888 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: read_amp_bytes_per_bit: 0 2024-09-17T11:48:07.888 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: format_version: 4 2024-09-17T11:48:07.888 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: enable_index_compression: 1 2024-09-17T11:48:07.888 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: block_align: 0 2024-09-17T11:48:07.888 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.write_buffer_size: 33554432 2024-09-17T11:48:07.889 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_write_buffer_number: 2 2024-09-17T11:48:07.889 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.compression: NoCompression 2024-09-17T11:48:07.891 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.bottommost_compression: Disabled 2024-09-17T11:48:07.891 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.prefix_extractor: nullptr 2024-09-17T11:48:07.892 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T11:48:07.892 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.num_levels: 7 2024-09-17T11:48:07.892 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T11:48:07.892 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T11:48:07.892 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T11:48:07.892 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T11:48:07.892 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T11:48:07.892 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T11:48:07.892 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T11:48:07.892 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T11:48:07.892 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-17T11:48:07.892 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T11:48:07.892 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T11:48:07.892 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.compression_opts.level: 32767 2024-09-17T11:48:07.893 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.compression_opts.strategy: 0 2024-09-17T11:48:07.893 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T11:48:07.893 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T11:48:07.893 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-17T11:48:07.893 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.compression_opts.enabled: false 2024-09-17T11:48:07.893 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T11:48:07.893 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T11:48:07.893 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T11:48:07.893 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.target_file_size_base: 67108864 2024-09-17T11:48:07.893 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T11:48:07.893 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T11:48:07.893 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T11:48:07.893 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T11:48:07.894 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T11:48:07.894 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T11:48:07.894 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T11:48:07.894 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T11:48:07.894 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T11:48:07.894 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T11:48:07.894 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T11:48:07.894 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T11:48:07.894 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T11:48:07.894 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.arena_block_size: 4194304 2024-09-17T11:48:07.894 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T11:48:07.894 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T11:48:07.894 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-17T11:48:07.894 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.disable_auto_compactions: 0 2024-09-17T11:48:07.894 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T11:48:07.895 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T11:48:07.895 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T11:48:07.895 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T11:48:07.895 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T11:48:07.895 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T11:48:07.895 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T11:48:07.895 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T11:48:07.895 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T11:48:07.895 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T11:48:07.895 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.table_properties_collectors: 2024-09-17T11:48:07.895 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.inplace_update_support: 0 2024-09-17T11:48:07.895 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T11:48:07.895 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T11:48:07.896 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T11:48:07.896 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T11:48:07.896 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.bloom_locality: 0 2024-09-17T11:48:07.896 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.max_successive_merges: 0 2024-09-17T11:48:07.897 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T11:48:07.897 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.paranoid_file_checks: 0 2024-09-17T11:48:07.897 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.force_consistency_checks: 1 2024-09-17T11:48:07.897 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.report_bg_io_stats: 0 2024-09-17T11:48:07.897 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.ttl: 2592000 2024-09-17T11:48:07.897 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T11:48:07.897 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.enable_blob_files: false 2024-09-17T11:48:07.897 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.min_blob_size: 0 2024-09-17T11:48:07.897 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.blob_file_size: 268435456 2024-09-17T11:48:07.897 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-17T11:48:07.897 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-17T11:48:07.897 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-17T11:48:07.897 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi045/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T11:48:07.898 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-17T11:48:07.898 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-17T11:48:07.898 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726573687625297, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-17T11:48:07.898 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-17T11:48:07.898 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-17T11:48:07.898 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726573687626063, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79397, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77661, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71970, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726573687, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "9e4595b0-aa55-453e-bdf4-bc0d9435fd9c", "db_session_id": "R3YOA8FQ8A9917HLC6KW"}} 2024-09-17T11:48:07.898 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-17T11:48:07.898 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726573687626389, "job": 1, "event": "recovery_finished"} 2024-09-17T11:48:07.898 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi045/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-17T11:48:07.898 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x560e45d96a80 2024-09-17T11:48:07.898 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: DB pointer 0x560e45da6000 2024-09-17T11:48:07.898 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-17T11:48:07.898 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-17T11:48:07.898 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: ** DB Stats ** 2024-09-17T11:48:07.899 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T11:48:07.899 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T11:48:07.899 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T11:48:07.899 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T11:48:07.899 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T11:48:07.899 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-17T11:48:07.899 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T11:48:07.899 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: 2024-09-17T11:48:07.899 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: ** Compaction Stats [default] ** 2024-09-17T11:48:07.899 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T11:48:07.899 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T11:48:07.899 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.8 0.00 0.00 1 0.001 0 0 2024-09-17T11:48:07.899 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.8 0.00 0.00 1 0.001 0 0 2024-09-17T11:48:07.899 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.8 0.00 0.00 1 0.001 0 0 2024-09-17T11:48:07.899 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: 2024-09-17T11:48:07.900 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: ** Compaction Stats [default] ** 2024-09-17T11:48:07.900 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T11:48:07.900 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T11:48:07.900 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 148.8 0.00 0.00 1 0.001 0 0 2024-09-17T11:48:07.900 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T11:48:07.900 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-17T11:48:07.900 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T11:48:07.900 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T11:48:07.900 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T11:48:07.900 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T11:48:07.900 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Cumulative compaction: 0.00 GB write, 24.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T11:48:07.900 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Interval compaction: 0.00 GB write, 24.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T11:48:07.900 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T11:48:07.900 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: 2024-09-17T11:48:07.901 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T11:48:07.901 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: 2024-09-17T11:48:07.901 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: ** Compaction Stats [default] ** 2024-09-17T11:48:07.901 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T11:48:07.901 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T11:48:07.901 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.8 0.00 0.00 1 0.001 0 0 2024-09-17T11:48:07.901 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.8 0.00 0.00 1 0.001 0 0 2024-09-17T11:48:07.901 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-17T11:48:07.901 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: 2024-09-17T11:48:07.901 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: ** Compaction Stats [default] ** 2024-09-17T11:48:07.901 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T11:48:07.901 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T11:48:07.901 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 148.8 0.00 0.00 1 0.001 0 0 2024-09-17T11:48:07.901 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T11:48:07.902 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-17T11:48:07.902 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T11:48:07.902 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T11:48:07.902 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T11:48:07.902 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T11:48:07.902 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Cumulative compaction: 0.00 GB write, 24.28 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T11:48:07.902 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T11:48:07.902 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T11:48:07.902 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: 2024-09-17T11:48:07.902 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T11:48:07.902 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: starting mon.smithi045 rank 0 at public addrs [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] at bind addrs [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi045 fsid a12537de-74ea-11ef-bceb-c7b262605968 2024-09-17T11:48:07.902 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: mon.smithi045@-1(???) e1 preinit fsid a12537de-74ea-11ef-bceb-c7b262605968 2024-09-17T11:48:07.902 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: mon.smithi045@-1(???).mds e1 new map 2024-09-17T11:48:07.902 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: mon.smithi045@-1(???).mds e1 print_map 2024-09-17T11:48:07.902 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: e1 2024-09-17T11:48:07.903 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T11:48:07.903 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T11:48:07.903 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: legacy client fscid: -1 2024-09-17T11:48:07.903 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: 2024-09-17T11:48:07.903 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: No filesystems configured 2024-09-17T11:48:07.903 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: mon.smithi045@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-17T11:48:07.903 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: mon.smithi045@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T11:48:07.903 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: mon.smithi045@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T11:48:07.903 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: mon.smithi045@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T11:48:07.903 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: mon.smithi045@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-17T11:48:07.903 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 bash[23718]: c4256bdf51b7d955c4309544040231c06bc9b4544fcc6131c1d2d633fb0ddccc 2024-09-17T11:48:07.903 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 systemd[1]: Started Ceph mon.smithi045 for a12537de-74ea-11ef-bceb-c7b262605968. 2024-09-17T11:48:07.903 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: mon.smithi045 is new leader, mons smithi045 in quorum (ranks 0) 2024-09-17T11:48:07.903 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: monmap e1: 1 mons at {smithi045=[v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0]} 2024-09-17T11:48:07.903 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: fsmap 2024-09-17T11:48:07.904 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: osdmap e1: 0 total, 0 up, 0 in 2024-09-17T11:48:07.904 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:07 smithi045 ceph-mon[23737]: mgrmap e1: no daemons active 2024-09-17T11:48:08.723 INFO:teuthology.orchestra.run.smithi045.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-17T11:48:08.723 INFO:teuthology.orchestra.run.smithi045.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-17T11:48:08.723 INFO:teuthology.orchestra.run.smithi045.stderr:Creating mgr... 2024-09-17T11:48:08.723 INFO:teuthology.orchestra.run.smithi045.stderr:Verifying port 9283 ... 2024-09-17T11:48:08.917 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl: Failed to reset failed state of unit ceph-a12537de-74ea-11ef-bceb-c7b262605968@mgr.smithi045.jtwyrp.service: Unit ceph-a12537de-74ea-11ef-bceb-c7b262605968@mgr.smithi045.jtwyrp.service not loaded. 2024-09-17T11:48:08.925 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl: Created symlink /etc/systemd/system/ceph-a12537de-74ea-11ef-bceb-c7b262605968.target.wants/ceph-a12537de-74ea-11ef-bceb-c7b262605968@mgr.smithi045.jtwyrp.service → /etc/systemd/system/ceph-a12537de-74ea-11ef-bceb-c7b262605968@.service. 2024-09-17T11:48:09.330 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:09 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/68960002' entity='client.admin' 2024-09-17T11:48:09.620 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl: disabled 2024-09-17T11:48:09.628 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl: inactive 2024-09-17T11:48:09.629 INFO:teuthology.orchestra.run.smithi045.stderr:firewalld.service is not enabled 2024-09-17T11:48:09.629 INFO:teuthology.orchestra.run.smithi045.stderr:Not possible to enable service . firewalld.service is not available 2024-09-17T11:48:09.638 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl: disabled 2024-09-17T11:48:09.646 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl: inactive 2024-09-17T11:48:09.647 INFO:teuthology.orchestra.run.smithi045.stderr:firewalld.service is not enabled 2024-09-17T11:48:09.647 INFO:teuthology.orchestra.run.smithi045.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-17T11:48:09.647 INFO:teuthology.orchestra.run.smithi045.stderr:Waiting for mgr to start... 2024-09-17T11:48:09.647 INFO:teuthology.orchestra.run.smithi045.stderr:Waiting for mgr... 2024-09-17T11:48:10.153 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: 2024-09-17T11:48:10.153 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: { 2024-09-17T11:48:10.153 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "fsid": "a12537de-74ea-11ef-bceb-c7b262605968", 2024-09-17T11:48:10.153 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "health": { 2024-09-17T11:48:10.154 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-17T11:48:10.154 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "checks": {}, 2024-09-17T11:48:10.154 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "mutes": [] 2024-09-17T11:48:10.154 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: }, 2024-09-17T11:48:10.154 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-17T11:48:10.154 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "quorum": [ 2024-09-17T11:48:10.154 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: 0 2024-09-17T11:48:10.154 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: ], 2024-09-17T11:48:10.154 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-17T11:48:10.154 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "smithi045" 2024-09-17T11:48:10.154 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: ], 2024-09-17T11:48:10.154 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-09-17T11:48:10.154 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "monmap": { 2024-09-17T11:48:10.154 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T11:48:10.155 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-17T11:48:10.155 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-17T11:48:10.155 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: }, 2024-09-17T11:48:10.155 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "osdmap": { 2024-09-17T11:48:10.155 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T11:48:10.155 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-17T11:48:10.155 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-17T11:48:10.155 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-17T11:48:10.155 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-17T11:48:10.159 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-17T11:48:10.159 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-17T11:48:10.159 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: }, 2024-09-17T11:48:10.159 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "pgmap": { 2024-09-17T11:48:10.159 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-17T11:48:10.159 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-17T11:48:10.159 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-17T11:48:10.159 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-17T11:48:10.159 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-17T11:48:10.159 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-17T11:48:10.159 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-17T11:48:10.159 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-17T11:48:10.159 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: }, 2024-09-17T11:48:10.160 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "fsmap": { 2024-09-17T11:48:10.160 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T11:48:10.160 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-17T11:48:10.160 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-17T11:48:10.160 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: }, 2024-09-17T11:48:10.160 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-17T11:48:10.160 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "available": false, 2024-09-17T11:48:10.160 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-17T11:48:10.160 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "modules": [ 2024-09-17T11:48:10.160 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "iostat", 2024-09-17T11:48:10.160 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "nfs", 2024-09-17T11:48:10.160 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "restful" 2024-09-17T11:48:10.160 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: ], 2024-09-17T11:48:10.161 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "services": {} 2024-09-17T11:48:10.161 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: }, 2024-09-17T11:48:10.161 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "servicemap": { 2024-09-17T11:48:10.161 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T11:48:10.161 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "modified": "2024-09-17T11:48:03.166385+0000", 2024-09-17T11:48:10.161 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "services": {} 2024-09-17T11:48:10.161 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: }, 2024-09-17T11:48:10.161 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-17T11:48:10.161 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: } 2024-09-17T11:48:10.409 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:10 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/78540580' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-17T11:48:10.658 INFO:teuthology.orchestra.run.smithi045.stderr:mgr not available, waiting (1/15)... 2024-09-17T11:48:13.172 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: 2024-09-17T11:48:13.172 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: { 2024-09-17T11:48:13.173 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "fsid": "a12537de-74ea-11ef-bceb-c7b262605968", 2024-09-17T11:48:13.173 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "health": { 2024-09-17T11:48:13.173 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-17T11:48:13.173 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "checks": {}, 2024-09-17T11:48:13.173 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "mutes": [] 2024-09-17T11:48:13.173 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: }, 2024-09-17T11:48:13.173 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-17T11:48:13.173 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "quorum": [ 2024-09-17T11:48:13.173 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: 0 2024-09-17T11:48:13.173 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: ], 2024-09-17T11:48:13.173 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-17T11:48:13.173 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "smithi045" 2024-09-17T11:48:13.173 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: ], 2024-09-17T11:48:13.174 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-09-17T11:48:13.174 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "monmap": { 2024-09-17T11:48:13.174 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T11:48:13.174 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-17T11:48:13.174 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-17T11:48:13.174 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: }, 2024-09-17T11:48:13.174 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "osdmap": { 2024-09-17T11:48:13.174 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T11:48:13.174 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-17T11:48:13.175 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-17T11:48:13.175 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-17T11:48:13.175 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-17T11:48:13.178 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-17T11:48:13.178 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-17T11:48:13.178 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: }, 2024-09-17T11:48:13.178 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "pgmap": { 2024-09-17T11:48:13.178 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-17T11:48:13.179 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-17T11:48:13.179 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-17T11:48:13.179 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-17T11:48:13.179 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-17T11:48:13.179 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-17T11:48:13.179 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-17T11:48:13.179 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-17T11:48:13.179 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: }, 2024-09-17T11:48:13.179 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "fsmap": { 2024-09-17T11:48:13.179 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T11:48:13.179 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-17T11:48:13.179 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-17T11:48:13.179 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: }, 2024-09-17T11:48:13.179 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-17T11:48:13.180 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "available": false, 2024-09-17T11:48:13.180 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-17T11:48:13.180 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "modules": [ 2024-09-17T11:48:13.180 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "iostat", 2024-09-17T11:48:13.180 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "nfs", 2024-09-17T11:48:13.180 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "restful" 2024-09-17T11:48:13.180 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: ], 2024-09-17T11:48:13.180 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "services": {} 2024-09-17T11:48:13.180 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: }, 2024-09-17T11:48:13.180 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "servicemap": { 2024-09-17T11:48:13.180 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T11:48:13.180 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "modified": "2024-09-17T11:48:03.166385+0000", 2024-09-17T11:48:13.181 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "services": {} 2024-09-17T11:48:13.181 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: }, 2024-09-17T11:48:13.181 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-17T11:48:13.181 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: } 2024-09-17T11:48:13.824 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:13 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/3638063137' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-17T11:48:16.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:15 smithi045 ceph-mon[23737]: Activating manager daemon smithi045.jtwyrp 2024-09-17T11:48:16.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:15 smithi045 ceph-mon[23737]: mgrmap e2: smithi045.jtwyrp(active, starting, since 0.00279368s) 2024-09-17T11:48:16.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:15 smithi045 ceph-mon[23737]: from='mgr.14100 172.21.15.45:0/4196999762' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T11:48:16.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:15 smithi045 ceph-mon[23737]: from='mgr.14100 172.21.15.45:0/4196999762' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T11:48:16.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:15 smithi045 ceph-mon[23737]: from='mgr.14100 172.21.15.45:0/4196999762' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T11:48:16.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:15 smithi045 ceph-mon[23737]: from='mgr.14100 172.21.15.45:0/4196999762' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi045"}]: dispatch 2024-09-17T11:48:16.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:15 smithi045 ceph-mon[23737]: from='mgr.14100 172.21.15.45:0/4196999762' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mgr metadata", "who": "smithi045.jtwyrp", "id": "smithi045.jtwyrp"}]: dispatch 2024-09-17T11:48:16.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:15 smithi045 ceph-mon[23737]: Manager daemon smithi045.jtwyrp is now available 2024-09-17T11:48:16.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:15 smithi045 ceph-mon[23737]: from='mgr.14100 172.21.15.45:0/4196999762' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:48:16.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:15 smithi045 ceph-mon[23737]: from='mgr.14100 172.21.15.45:0/4196999762' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:48:16.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:15 smithi045 ceph-mon[23737]: from='mgr.14100 172.21.15.45:0/4196999762' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:48:16.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:15 smithi045 ceph-mon[23737]: from='mgr.14100 172.21.15.45:0/4196999762' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:48:16.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:15 smithi045 ceph-mon[23737]: from='mgr.14100 172.21.15.45:0/4196999762' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:48:17.100 INFO:teuthology.orchestra.run.smithi045.stderr:mgr not available, waiting (2/15)... 2024-09-17T11:48:18.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:17 smithi045 ceph-mon[23737]: mgrmap e3: smithi045.jtwyrp(active, since 1.0065s) 2024-09-17T11:48:20.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:19 smithi045 ceph-mon[23737]: mgrmap e4: smithi045.jtwyrp(active, since 2s) 2024-09-17T11:48:23.351 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: 2024-09-17T11:48:23.352 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: { 2024-09-17T11:48:23.352 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "fsid": "a12537de-74ea-11ef-bceb-c7b262605968", 2024-09-17T11:48:23.352 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "health": { 2024-09-17T11:48:23.352 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-17T11:48:23.352 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "checks": {}, 2024-09-17T11:48:23.352 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "mutes": [] 2024-09-17T11:48:23.352 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: }, 2024-09-17T11:48:23.352 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-17T11:48:23.352 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "quorum": [ 2024-09-17T11:48:23.352 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: 0 2024-09-17T11:48:23.352 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: ], 2024-09-17T11:48:23.352 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-17T11:48:23.353 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "smithi045" 2024-09-17T11:48:23.353 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: ], 2024-09-17T11:48:23.353 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "quorum_age": 15, 2024-09-17T11:48:23.353 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "monmap": { 2024-09-17T11:48:23.353 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T11:48:23.353 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-17T11:48:23.353 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-17T11:48:23.353 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: }, 2024-09-17T11:48:23.353 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "osdmap": { 2024-09-17T11:48:23.353 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T11:48:23.353 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-17T11:48:23.353 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-17T11:48:23.353 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-17T11:48:23.353 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-17T11:48:23.357 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-17T11:48:23.357 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-17T11:48:23.357 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: }, 2024-09-17T11:48:23.357 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "pgmap": { 2024-09-17T11:48:23.357 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-17T11:48:23.357 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-17T11:48:23.357 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-17T11:48:23.357 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-17T11:48:23.358 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-17T11:48:23.358 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-17T11:48:23.358 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-17T11:48:23.358 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-17T11:48:23.358 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: }, 2024-09-17T11:48:23.358 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "fsmap": { 2024-09-17T11:48:23.358 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T11:48:23.358 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-17T11:48:23.358 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-17T11:48:23.358 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: }, 2024-09-17T11:48:23.358 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-17T11:48:23.358 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "available": true, 2024-09-17T11:48:23.358 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-17T11:48:23.358 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "modules": [ 2024-09-17T11:48:23.358 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "iostat", 2024-09-17T11:48:23.359 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "nfs", 2024-09-17T11:48:23.359 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "restful" 2024-09-17T11:48:23.359 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: ], 2024-09-17T11:48:23.359 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "services": {} 2024-09-17T11:48:23.359 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: }, 2024-09-17T11:48:23.359 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "servicemap": { 2024-09-17T11:48:23.359 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T11:48:23.359 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "modified": "2024-09-17T11:48:03.166385+0000", 2024-09-17T11:48:23.359 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "services": {} 2024-09-17T11:48:23.359 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: }, 2024-09-17T11:48:23.359 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-17T11:48:23.359 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: } 2024-09-17T11:48:23.948 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:23 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/3740639100' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-17T11:48:24.251 INFO:teuthology.orchestra.run.smithi045.stderr:mgr is available 2024-09-17T11:48:24.839 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: 2024-09-17T11:48:24.839 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: [global] 2024-09-17T11:48:24.840 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: fsid = a12537de-74ea-11ef-bceb-c7b262605968 2024-09-17T11:48:24.840 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-17T11:48:24.840 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-17T11:48:24.840 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-17T11:48:24.840 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-17T11:48:24.840 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: 2024-09-17T11:48:24.840 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: [mgr] 2024-09-17T11:48:24.840 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-17T11:48:24.840 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: 2024-09-17T11:48:24.840 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: [osd] 2024-09-17T11:48:24.841 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-17T11:48:24.841 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-17T11:48:25.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:24 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/1810183056' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-17T11:48:25.374 INFO:teuthology.orchestra.run.smithi045.stderr:Enabling cephadm module... 2024-09-17T11:48:26.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:26 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/418470691' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-17T11:48:27.290 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:27 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/418470691' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-17T11:48:27.290 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:27 smithi045 ceph-mon[23737]: mgrmap e5: smithi045.jtwyrp(active, since 10s) 2024-09-17T11:48:27.319 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: { 2024-09-17T11:48:27.319 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-17T11:48:27.319 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "available": true, 2024-09-17T11:48:27.319 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "active_name": "smithi045.jtwyrp", 2024-09-17T11:48:27.320 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-17T11:48:27.320 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: } 2024-09-17T11:48:27.880 INFO:teuthology.orchestra.run.smithi045.stderr:Waiting for the mgr to restart... 2024-09-17T11:48:27.880 INFO:teuthology.orchestra.run.smithi045.stderr:Waiting for mgr epoch 5... 2024-09-17T11:48:28.133 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:28 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/3439180278' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-17T11:48:33.440 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: { 2024-09-17T11:48:33.440 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-17T11:48:33.440 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "initialized": true 2024-09-17T11:48:33.440 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: } 2024-09-17T11:48:33.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:33 smithi045 ceph-mon[23737]: Active manager daemon smithi045.jtwyrp restarted 2024-09-17T11:48:33.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:33 smithi045 ceph-mon[23737]: Activating manager daemon smithi045.jtwyrp 2024-09-17T11:48:33.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:33 smithi045 ceph-mon[23737]: osdmap e2: 0 total, 0 up, 0 in 2024-09-17T11:48:33.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:33 smithi045 ceph-mon[23737]: mgrmap e6: smithi045.jtwyrp(active, starting, since 0.0524248s) 2024-09-17T11:48:33.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:33 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi045"}]: dispatch 2024-09-17T11:48:33.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:33 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mgr metadata", "who": "smithi045.jtwyrp", "id": "smithi045.jtwyrp"}]: dispatch 2024-09-17T11:48:33.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:33 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T11:48:33.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:33 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T11:48:33.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:33 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T11:48:33.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:33 smithi045 ceph-mon[23737]: Manager daemon smithi045.jtwyrp is now available 2024-09-17T11:48:33.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:33 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:48:33.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:33 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:48:33.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:33 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:48:33.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:33 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:48:33.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:33 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:48:33.591 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:33 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:48:33.591 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:33 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:48:33.928 INFO:teuthology.orchestra.run.smithi045.stderr:mgr epoch 5 is available 2024-09-17T11:48:33.928 INFO:teuthology.orchestra.run.smithi045.stderr:Setting orchestrator backend to cephadm... 2024-09-17T11:48:34.511 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:34 smithi045 ceph-mon[23737]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-17T11:48:34.511 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:34 smithi045 ceph-mon[23737]: mgrmap e7: smithi045.jtwyrp(active, since 1.05491s) 2024-09-17T11:48:34.511 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:34 smithi045 ceph-mon[23737]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-17T11:48:35.704 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:35 smithi045 ceph-mon[23737]: mgrmap e8: smithi045.jtwyrp(active, since 2s) 2024-09-17T11:48:35.704 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:35 smithi045 ceph-mon[23737]: [17/Sep/2024:11:48:34] ENGINE Bus STARTING 2024-09-17T11:48:35.704 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:35 smithi045 ceph-mon[23737]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:48:35.704 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:35 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:48:35.704 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:35 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:48:35.704 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:35 smithi045 ceph-mon[23737]: [17/Sep/2024:11:48:34] ENGINE Serving on https://172.21.15.45:7150 2024-09-17T11:48:35.704 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:35 smithi045 ceph-mon[23737]: [17/Sep/2024:11:48:34] ENGINE Bus STARTED 2024-09-17T11:48:35.704 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:35 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:48:35.704 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:35 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:48:35.705 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: value unchanged 2024-09-17T11:48:36.258 INFO:teuthology.orchestra.run.smithi045.stderr:Generating ssh key... 2024-09-17T11:48:36.513 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:36 smithi045 ceph-mon[23737]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:48:38.195 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:38 smithi045 ceph-mon[23737]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:48:38.196 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:38 smithi045 ceph-mon[23737]: Generating ssh key... 2024-09-17T11:48:38.196 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:38 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:48:38.196 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:38 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:48:38.196 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDnldyQroR8KRTw9CMvYcV7uy/teisR9x3gOGK/DWqzgBNfbdoULSoxTCXhAtKh0dqvR8TkJ9/Lbt2lovo73L1o1XS81s9ApllH+qUfMiQW8DN4E0qYWGvIes1RWpSIhRG60Jw1wHNclYP5z4XeVBukUPqqRtTbeud34i19re+GFXr/K7ULcXa3IFs522/90+WmJIz+kuizlGkM0FfAePRgiRP10kN0YaKDMBRlZ/hz909GFjCUAWuaMrdH4Zxnl1Dm4D/WbOGslBhfG0y2qyQ7EpQyRHsIf2Ydtz34PXt2RlegeUV0HW7NriSMq/G9abZVZLKc+NSiyxFwgITbqGXgjG+Ac3hKdBOnMcoHEPAUqgw2T6MiiuznQSsyrndsgXk3d/T0i0IHlY3KvccHuPyZ4e+Qu1iLdnkzJ/1n/Ik+N8P9Q4WBA/r5rzazRECMSf3S2H2NTCQsuXFXQL+Frn8L4kqsLLEE89WJEuWze+tHGtLK+JbnRx5YrtnKENGszgs= ceph-a12537de-74ea-11ef-bceb-c7b262605968 2024-09-17T11:48:38.772 INFO:teuthology.orchestra.run.smithi045.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-17T11:48:38.772 INFO:teuthology.orchestra.run.smithi045.stderr:Adding key to root@localhost authorized_keys... 2024-09-17T11:48:38.777 INFO:teuthology.orchestra.run.smithi045.stderr:Adding host smithi045... 2024-09-17T11:48:39.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:39 smithi045 ceph-mon[23737]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:48:40.316 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: Added host 'smithi045' with addr '172.21.15.45' 2024-09-17T11:48:40.328 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:40 smithi045 ceph-mon[23737]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi045", "addr": "172.21.15.45", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:48:40.328 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:40 smithi045 ceph-mon[23737]: Deploying cephadm binary to smithi045 2024-09-17T11:48:40.938 INFO:teuthology.orchestra.run.smithi045.stderr:Deploying mon service with default placement... 2024-09-17T11:48:41.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:41 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:48:41.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:41 smithi045 ceph-mon[23737]: Added host smithi045 2024-09-17T11:48:41.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:41 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:48:41.593 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-17T11:48:42.392 INFO:teuthology.orchestra.run.smithi045.stderr:Deploying mgr service with default placement... 2024-09-17T11:48:42.727 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:42 smithi045 ceph-mon[23737]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:48:42.727 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:42 smithi045 ceph-mon[23737]: Saving service mon spec with placement count:5 2024-09-17T11:48:42.727 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:42 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:48:43.058 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-17T11:48:43.809 INFO:teuthology.orchestra.run.smithi045.stderr:Deploying crash service with default placement... 2024-09-17T11:48:44.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:44 smithi045 ceph-mon[23737]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:48:44.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:44 smithi045 ceph-mon[23737]: Saving service mgr spec with placement count:2 2024-09-17T11:48:44.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:44 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:48:45.020 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: Scheduled crash update... 2024-09-17T11:48:45.594 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:45 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:48:45.594 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:45 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:48:45.595 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:45 smithi045 ceph-mon[23737]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:48:45.595 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:45 smithi045 ceph-mon[23737]: Saving service crash spec with placement * 2024-09-17T11:48:45.595 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:45 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:48:45.595 INFO:teuthology.orchestra.run.smithi045.stderr:Deploying prometheus service with default placement... 2024-09-17T11:48:46.216 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-09-17T11:48:47.262 INFO:teuthology.orchestra.run.smithi045.stderr:Deploying grafana service with default placement... 2024-09-17T11:48:47.500 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:47 smithi045 ceph-mon[23737]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:48:47.500 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:47 smithi045 ceph-mon[23737]: Saving service prometheus spec with placement count:1 2024-09-17T11:48:47.500 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:47 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:48:47.500 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:47 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:48:47.917 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-09-17T11:48:48.469 INFO:teuthology.orchestra.run.smithi045.stderr:Deploying node-exporter service with default placement... 2024-09-17T11:48:48.977 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:48 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:48:48.977 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:48 smithi045 ceph-mon[23737]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:48:48.978 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:48 smithi045 ceph-mon[23737]: Saving service grafana spec with placement count:1 2024-09-17T11:48:48.978 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:48 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:48:50.365 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-09-17T11:48:50.981 INFO:teuthology.orchestra.run.smithi045.stderr:Deploying alertmanager service with default placement... 2024-09-17T11:48:51.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:51 smithi045 ceph-mon[23737]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:48:51.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:51 smithi045 ceph-mon[23737]: Saving service node-exporter spec with placement * 2024-09-17T11:48:51.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:51 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:48:51.623 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-09-17T11:48:52.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:52 smithi045 ceph-mon[23737]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:48:52.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:52 smithi045 ceph-mon[23737]: Saving service alertmanager spec with placement count:1 2024-09-17T11:48:52.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:52 smithi045 ceph-mon[23737]: from='mgr.14118 172.21.15.45:0/3300112509' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:48:54.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:53 smithi045 ceph-mon[23737]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:48:54.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:53 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/2389566515' entity='client.admin' 2024-09-17T11:48:54.693 INFO:teuthology.orchestra.run.smithi045.stderr:Enabling the dashboard module... 2024-09-17T11:48:55.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:55 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/2916262938' entity='client.admin' 2024-09-17T11:48:56.359 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:56 smithi045 ceph-mon[23737]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:48:56.359 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:56 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/3987867400' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-17T11:48:57.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:57 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/3987867400' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-17T11:48:57.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:57 smithi045 ceph-mon[23737]: mgrmap e9: smithi045.jtwyrp(active, since 23s) 2024-09-17T11:48:57.394 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: { 2024-09-17T11:48:57.395 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-17T11:48:57.395 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "available": true, 2024-09-17T11:48:57.395 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "active_name": "smithi045.jtwyrp", 2024-09-17T11:48:57.395 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-17T11:48:57.395 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: } 2024-09-17T11:48:58.073 INFO:teuthology.orchestra.run.smithi045.stderr:Waiting for the mgr to restart... 2024-09-17T11:48:58.073 INFO:teuthology.orchestra.run.smithi045.stderr:Waiting for mgr epoch 9... 2024-09-17T11:48:58.331 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:48:58 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/543470820' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-17T11:49:03.515 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: { 2024-09-17T11:49:03.515 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-17T11:49:03.516 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: "initialized": true 2024-09-17T11:49:03.516 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: } 2024-09-17T11:49:03.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:03 smithi045 ceph-mon[23737]: Active manager daemon smithi045.jtwyrp restarted 2024-09-17T11:49:03.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:03 smithi045 ceph-mon[23737]: Activating manager daemon smithi045.jtwyrp 2024-09-17T11:49:03.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:03 smithi045 ceph-mon[23737]: osdmap e3: 0 total, 0 up, 0 in 2024-09-17T11:49:03.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:03 smithi045 ceph-mon[23737]: mgrmap e10: smithi045.jtwyrp(active, starting, since 0.0526218s) 2024-09-17T11:49:03.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:03 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi045"}]: dispatch 2024-09-17T11:49:03.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:03 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mgr metadata", "who": "smithi045.jtwyrp", "id": "smithi045.jtwyrp"}]: dispatch 2024-09-17T11:49:03.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:03 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T11:49:03.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:03 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T11:49:03.841 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:03 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T11:49:03.841 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:03 smithi045 ceph-mon[23737]: Manager daemon smithi045.jtwyrp is now available 2024-09-17T11:49:03.841 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:03 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:49:03.841 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:03 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:49:03.841 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:03 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:49:03.841 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:03 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:49:03.841 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:03 smithi045 ceph-mon[23737]: [17/Sep/2024:11:49:02] ENGINE Bus STARTING 2024-09-17T11:49:03.841 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:03 smithi045 ceph-mon[23737]: [17/Sep/2024:11:49:02] ENGINE Serving on https://172.21.15.45:7150 2024-09-17T11:49:03.841 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:03 smithi045 ceph-mon[23737]: [17/Sep/2024:11:49:02] ENGINE Bus STARTED 2024-09-17T11:49:03.841 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:03 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:04.258 INFO:teuthology.orchestra.run.smithi045.stderr:mgr epoch 9 is available 2024-09-17T11:49:04.259 INFO:teuthology.orchestra.run.smithi045.stderr:Generating a dashboard self-signed certificate... 2024-09-17T11:49:04.792 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:04 smithi045 ceph-mon[23737]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-17T11:49:04.792 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:04 smithi045 ceph-mon[23737]: mgrmap e11: smithi045.jtwyrp(active, since 1.05521s) 2024-09-17T11:49:04.793 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:04 smithi045 ceph-mon[23737]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-17T11:49:04.793 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:04 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:05.101 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-17T11:49:05.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:05 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:05.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:05 smithi045 ceph-mon[23737]: mgrmap e12: smithi045.jtwyrp(active, since 2s) 2024-09-17T11:49:05.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:05 smithi045 ceph-mon[23737]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:49:05.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:05 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:05.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:05 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:06.195 INFO:teuthology.orchestra.run.smithi045.stderr:Creating initial admin user... 2024-09-17T11:49:07.557 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$6K3Ad3OB.eN/XPTt0LMTNerG46leWTaqa/x8PavcnLPvLXhPII3.i", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726573747, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-17T11:49:08.238 INFO:teuthology.orchestra.run.smithi045.stderr:Fetching dashboard port number... 2024-09-17T11:49:08.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:08 smithi045 ceph-mon[23737]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:49:08.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:08 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:08.882 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: 8443 2024-09-17T11:49:09.565 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl: disabled 2024-09-17T11:49:09.574 INFO:teuthology.orchestra.run.smithi045.stderr:systemctl: inactive 2024-09-17T11:49:09.575 INFO:teuthology.orchestra.run.smithi045.stderr:firewalld.service is not enabled 2024-09-17T11:49:09.575 INFO:teuthology.orchestra.run.smithi045.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-17T11:49:09.577 INFO:teuthology.orchestra.run.smithi045.stderr:Ceph Dashboard is now available at: 2024-09-17T11:49:09.578 INFO:teuthology.orchestra.run.smithi045.stderr: 2024-09-17T11:49:09.578 INFO:teuthology.orchestra.run.smithi045.stderr: URL: https://smithi045.front.sepia.ceph.com:8443/ 2024-09-17T11:49:09.578 INFO:teuthology.orchestra.run.smithi045.stderr: User: admin 2024-09-17T11:49:09.578 INFO:teuthology.orchestra.run.smithi045.stderr: Password: e6m9tjh7hg 2024-09-17T11:49:09.578 INFO:teuthology.orchestra.run.smithi045.stderr: 2024-09-17T11:49:09.578 INFO:teuthology.orchestra.run.smithi045.stderr:Enabling autotune for osd_memory_target 2024-09-17T11:49:09.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:09 smithi045 ceph-mon[23737]: mgrmap e13: smithi045.jtwyrp(active, since 6s) 2024-09-17T11:49:09.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:09 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/3718159657' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-17T11:49:11.653 INFO:teuthology.orchestra.run.smithi045.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-17T11:49:12.251 INFO:teuthology.orchestra.run.smithi045.stderr:You can access the Ceph CLI with: 2024-09-17T11:49:12.251 INFO:teuthology.orchestra.run.smithi045.stderr: 2024-09-17T11:49:12.251 INFO:teuthology.orchestra.run.smithi045.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-17T11:49:12.251 INFO:teuthology.orchestra.run.smithi045.stderr: 2024-09-17T11:49:12.251 INFO:teuthology.orchestra.run.smithi045.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-17T11:49:12.252 INFO:teuthology.orchestra.run.smithi045.stderr: 2024-09-17T11:49:12.252 INFO:teuthology.orchestra.run.smithi045.stderr: ceph telemetry on 2024-09-17T11:49:12.252 INFO:teuthology.orchestra.run.smithi045.stderr: 2024-09-17T11:49:12.252 INFO:teuthology.orchestra.run.smithi045.stderr:For more information see: 2024-09-17T11:49:12.252 INFO:teuthology.orchestra.run.smithi045.stderr: 2024-09-17T11:49:12.252 INFO:teuthology.orchestra.run.smithi045.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-17T11:49:12.252 INFO:teuthology.orchestra.run.smithi045.stderr: 2024-09-17T11:49:12.252 INFO:teuthology.orchestra.run.smithi045.stderr:Bootstrap complete. 2024-09-17T11:49:12.291 INFO:tasks.cephadm:Fetching config... 2024-09-17T11:49:12.292 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2024-09-17T11:49:12.292 DEBUG:teuthology.orchestra.run.smithi045:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-17T11:49:12.310 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-17T11:49:12.310 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2024-09-17T11:49:12.310 DEBUG:teuthology.orchestra.run.smithi045:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-17T11:49:12.368 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-17T11:49:12.368 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2024-09-17T11:49:12.368 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd if=/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/keyring of=/dev/stdout 2024-09-17T11:49:12.436 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-17T11:49:12.436 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2024-09-17T11:49:12.436 DEBUG:teuthology.orchestra.run.smithi045:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-17T11:49:12.494 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-17T11:49:12.495 DEBUG:teuthology.orchestra.run.smithi045:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDnldyQroR8KRTw9CMvYcV7uy/teisR9x3gOGK/DWqzgBNfbdoULSoxTCXhAtKh0dqvR8TkJ9/Lbt2lovo73L1o1XS81s9ApllH+qUfMiQW8DN4E0qYWGvIes1RWpSIhRG60Jw1wHNclYP5z4XeVBukUPqqRtTbeud34i19re+GFXr/K7ULcXa3IFs522/90+WmJIz+kuizlGkM0FfAePRgiRP10kN0YaKDMBRlZ/hz909GFjCUAWuaMrdH4Zxnl1Dm4D/WbOGslBhfG0y2qyQ7EpQyRHsIf2Ydtz34PXt2RlegeUV0HW7NriSMq/G9abZVZLKc+NSiyxFwgITbqGXgjG+Ac3hKdBOnMcoHEPAUqgw2T6MiiuznQSsyrndsgXk3d/T0i0IHlY3KvccHuPyZ4e+Qu1iLdnkzJ/1n/Ik+N8P9Q4WBA/r5rzazRECMSf3S2H2NTCQsuXFXQL+Frn8L4kqsLLEE89WJEuWze+tHGtLK+JbnRx5YrtnKENGszgs= ceph-a12537de-74ea-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-17T11:49:12.576 INFO:teuthology.orchestra.run.smithi045.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDnldyQroR8KRTw9CMvYcV7uy/teisR9x3gOGK/DWqzgBNfbdoULSoxTCXhAtKh0dqvR8TkJ9/Lbt2lovo73L1o1XS81s9ApllH+qUfMiQW8DN4E0qYWGvIes1RWpSIhRG60Jw1wHNclYP5z4XeVBukUPqqRtTbeud34i19re+GFXr/K7ULcXa3IFs522/90+WmJIz+kuizlGkM0FfAePRgiRP10kN0YaKDMBRlZ/hz909GFjCUAWuaMrdH4Zxnl1Dm4D/WbOGslBhfG0y2qyQ7EpQyRHsIf2Ydtz34PXt2RlegeUV0HW7NriSMq/G9abZVZLKc+NSiyxFwgITbqGXgjG+Ac3hKdBOnMcoHEPAUqgw2T6MiiuznQSsyrndsgXk3d/T0i0IHlY3KvccHuPyZ4e+Qu1iLdnkzJ/1n/Ik+N8P9Q4WBA/r5rzazRECMSf3S2H2NTCQsuXFXQL+Frn8L4kqsLLEE89WJEuWze+tHGtLK+JbnRx5YrtnKENGszgs= ceph-a12537de-74ea-11ef-bceb-c7b262605968 2024-09-17T11:49:12.587 DEBUG:teuthology.orchestra.run.smithi165:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDnldyQroR8KRTw9CMvYcV7uy/teisR9x3gOGK/DWqzgBNfbdoULSoxTCXhAtKh0dqvR8TkJ9/Lbt2lovo73L1o1XS81s9ApllH+qUfMiQW8DN4E0qYWGvIes1RWpSIhRG60Jw1wHNclYP5z4XeVBukUPqqRtTbeud34i19re+GFXr/K7ULcXa3IFs522/90+WmJIz+kuizlGkM0FfAePRgiRP10kN0YaKDMBRlZ/hz909GFjCUAWuaMrdH4Zxnl1Dm4D/WbOGslBhfG0y2qyQ7EpQyRHsIf2Ydtz34PXt2RlegeUV0HW7NriSMq/G9abZVZLKc+NSiyxFwgITbqGXgjG+Ac3hKdBOnMcoHEPAUqgw2T6MiiuznQSsyrndsgXk3d/T0i0IHlY3KvccHuPyZ4e+Qu1iLdnkzJ/1n/Ik+N8P9Q4WBA/r5rzazRECMSf3S2H2NTCQsuXFXQL+Frn8L4kqsLLEE89WJEuWze+tHGtLK+JbnRx5YrtnKENGszgs= ceph-a12537de-74ea-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-17T11:49:12.647 INFO:teuthology.orchestra.run.smithi165.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDnldyQroR8KRTw9CMvYcV7uy/teisR9x3gOGK/DWqzgBNfbdoULSoxTCXhAtKh0dqvR8TkJ9/Lbt2lovo73L1o1XS81s9ApllH+qUfMiQW8DN4E0qYWGvIes1RWpSIhRG60Jw1wHNclYP5z4XeVBukUPqqRtTbeud34i19re+GFXr/K7ULcXa3IFs522/90+WmJIz+kuizlGkM0FfAePRgiRP10kN0YaKDMBRlZ/hz909GFjCUAWuaMrdH4Zxnl1Dm4D/WbOGslBhfG0y2qyQ7EpQyRHsIf2Ydtz34PXt2RlegeUV0HW7NriSMq/G9abZVZLKc+NSiyxFwgITbqGXgjG+Ac3hKdBOnMcoHEPAUqgw2T6MiiuznQSsyrndsgXk3d/T0i0IHlY3KvccHuPyZ4e+Qu1iLdnkzJ/1n/Ik+N8P9Q4WBA/r5rzazRECMSf3S2H2NTCQsuXFXQL+Frn8L4kqsLLEE89WJEuWze+tHGtLK+JbnRx5YrtnKENGszgs= ceph-a12537de-74ea-11ef-bceb-c7b262605968 2024-09-17T11:49:12.659 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-17T11:49:12.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:12 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/4159616622' entity='client.admin' 2024-09-17T11:49:15.881 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-17T11:49:15.881 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-17T11:49:16.533 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:16 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/163624076' entity='client.admin' 2024-09-17T11:49:18.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:18 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:18.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:18 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd/host:smithi045", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:49:18.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:18 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:18.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:18 smithi045 ceph-mon[23737]: Deploying daemon alertmanager.smithi045 on smithi045 2024-09-17T11:49:19.176 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi165 2024-09-17T11:49:19.177 DEBUG:teuthology.orchestra.run.smithi165:> set -ex 2024-09-17T11:49:19.177 DEBUG:teuthology.orchestra.run.smithi165:> dd of=/etc/ceph/ceph.conf 2024-09-17T11:49:19.196 DEBUG:teuthology.orchestra.run.smithi165:> set -ex 2024-09-17T11:49:19.196 DEBUG:teuthology.orchestra.run.smithi165:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-17T11:49:19.254 INFO:tasks.cephadm:Adding host smithi165 to orchestrator... 2024-09-17T11:49:19.254 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph orch host add smithi165 2024-09-17T11:49:19.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:19 smithi045 ceph-mon[23737]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:49:19.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:19 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:23.366 INFO:teuthology.orchestra.run.smithi045.stdout:Added host 'smithi165' with addr '172.21.15.165' 2024-09-17T11:49:23.767 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:23 smithi045 ceph-mon[23737]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi165", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:49:23.767 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:23 smithi045 ceph-mon[23737]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:49:23.767 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:23 smithi045 ceph-mon[23737]: Deploying cephadm binary to smithi165 2024-09-17T11:49:23.767 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:23 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:24.829 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:24 smithi045 ceph-mon[23737]: Added host smithi165 2024-09-17T11:49:25.430 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-17T11:49:25.711 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:25 smithi045 ceph-mon[23737]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:49:27.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:27 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:27.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:27 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi045", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T11:49:27.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:27 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi045", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-17T11:49:27.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:27 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:49:27.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:27 smithi045 ceph-mon[23737]: Deploying daemon crash.smithi045 on smithi045 2024-09-17T11:49:28.469 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:28 smithi045 ceph-mon[23737]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:49:28.469 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:28 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:29.037 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:49:29.037 INFO:teuthology.orchestra.run.smithi045.stdout:[{"addr": "172.21.15.45", "hostname": "smithi045", "labels": [], "status": ""}, {"addr": "172.21.15.165", "hostname": "smithi165", "labels": [], "status": ""}] 2024-09-17T11:49:29.975 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-17T11:49:29.975 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-17T11:49:30.292 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:30 smithi045 ceph-mon[23737]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:49:30.292 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:30 smithi045 ceph-mon[23737]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T11:49:30.292 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:30 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:30.292 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:30 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:30.292 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:30 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:30.292 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:30 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T11:49:30.292 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:30 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:31.402 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:31 smithi045 ceph-mon[23737]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T11:49:31.402 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:31 smithi045 ceph-mon[23737]: Deploying daemon grafana.smithi045 on smithi045 2024-09-17T11:49:32.142 INFO:teuthology.orchestra.run.smithi045.stderr:adjusted tunables profile to default 2024-09-17T11:49:32.471 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:32 smithi045 ceph-mon[23737]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:49:32.471 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:32 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/1470483078' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-17T11:49:32.782 INFO:tasks.cephadm:Adding mon.smithi045 on smithi045 2024-09-17T11:49:32.783 INFO:tasks.cephadm:Adding mon.smithi165 on smithi165 2024-09-17T11:49:32.783 DEBUG:teuthology.orchestra.run.smithi165:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph orch apply mon '2;smithi045:172.21.15.45=smithi045;smithi165:172.21.15.165=smithi165' 2024-09-17T11:49:33.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:33 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/1470483078' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-17T11:49:33.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:33 smithi045 ceph-mon[23737]: osdmap e4: 0 total, 0 up, 0 in 2024-09-17T11:49:33.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:33 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:34.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:34 smithi045 ceph-mon[23737]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:49:34.845 INFO:teuthology.orchestra.run.smithi165.stdout:Scheduled mon update... 2024-09-17T11:49:35.385 DEBUG:teuthology.orchestra.run.smithi165:mon.smithi165> sudo journalctl -f -n 0 -u ceph-a12537de-74ea-11ef-bceb-c7b262605968@mon.smithi165.service 2024-09-17T11:49:35.388 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T11:49:35.388 DEBUG:teuthology.orchestra.run.smithi165:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T11:49:36.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:35 smithi045 ceph-mon[23737]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:49:36.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:35 smithi045 ceph-mon[23737]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi045:172.21.15.45=smithi045;smithi165:172.21.15.165=smithi165", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:49:36.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:35 smithi045 ceph-mon[23737]: Saving service mon spec with placement smithi045:172.21.15.45=smithi045;smithi165:172.21.15.165=smithi165;count:2 2024-09-17T11:49:36.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:35 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:37.298 INFO:teuthology.orchestra.run.smithi165.stdout: 2024-09-17T11:49:37.298 INFO:teuthology.orchestra.run.smithi165.stdout:{"epoch":1,"fsid":"a12537de-74ea-11ef-bceb-c7b262605968","modified":"2024-09-17T11:48:00.407559Z","created":"2024-09-17T11:48:00.407559Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi045","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:3300","nonce":0},{"type":"v1","addr":"172.21.15.45:6789","nonce":0}]},"addr":"172.21.15.45:6789/0","public_addr":"172.21.15.45:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T11:49:37.300 INFO:teuthology.orchestra.run.smithi165.stderr:dumped monmap epoch 1 2024-09-17T11:49:38.188 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:37 smithi045 ceph-mon[23737]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:49:38.188 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:37 smithi045 ceph-mon[23737]: from='client.? 172.21.15.165:0/1196919074' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T11:49:38.849 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T11:49:38.849 DEBUG:teuthology.orchestra.run.smithi165:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T11:49:40.095 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:39 smithi045 ceph-mon[23737]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:49:40.657 INFO:teuthology.orchestra.run.smithi165.stdout: 2024-09-17T11:49:40.657 INFO:teuthology.orchestra.run.smithi165.stdout:{"epoch":1,"fsid":"a12537de-74ea-11ef-bceb-c7b262605968","modified":"2024-09-17T11:48:00.407559Z","created":"2024-09-17T11:48:00.407559Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi045","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:3300","nonce":0},{"type":"v1","addr":"172.21.15.45:6789","nonce":0}]},"addr":"172.21.15.45:6789/0","public_addr":"172.21.15.45:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T11:49:40.659 INFO:teuthology.orchestra.run.smithi165.stderr:dumped monmap epoch 1 2024-09-17T11:49:41.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:41 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:41.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:41 smithi045 ceph-mon[23737]: Deploying daemon node-exporter.smithi045 on smithi045 2024-09-17T11:49:41.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:41 smithi045 ceph-mon[23737]: from='client.? 172.21.15.165:0/2691153185' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T11:49:42.226 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T11:49:42.226 DEBUG:teuthology.orchestra.run.smithi165:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T11:49:42.534 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:42 smithi045 ceph-mon[23737]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:49:43.843 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:43 smithi045 ceph-mon[23737]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:49:43.843 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:43 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:43.843 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:43 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:43.843 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:43 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-17T11:49:44.102 INFO:teuthology.orchestra.run.smithi165.stdout: 2024-09-17T11:49:44.102 INFO:teuthology.orchestra.run.smithi165.stdout:{"epoch":1,"fsid":"a12537de-74ea-11ef-bceb-c7b262605968","modified":"2024-09-17T11:48:00.407559Z","created":"2024-09-17T11:48:00.407559Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi045","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:3300","nonce":0},{"type":"v1","addr":"172.21.15.45:6789","nonce":0}]},"addr":"172.21.15.45:6789/0","public_addr":"172.21.15.45:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T11:49:44.103 INFO:teuthology.orchestra.run.smithi165.stderr:dumped monmap epoch 1 2024-09-17T11:49:44.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:44 smithi045 ceph-mon[23737]: from='client.? 172.21.15.165:0/2807981989' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T11:49:44.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:44 smithi045 ceph-mon[23737]: from='mgr.14160 172.21.15.45:0/2759427758' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-17T11:49:44.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:44 smithi045 ceph-mon[23737]: mgrmap e14: smithi045.jtwyrp(active, since 41s) 2024-09-17T11:49:45.749 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T11:49:45.750 DEBUG:teuthology.orchestra.run.smithi165:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T11:49:47.552 INFO:teuthology.orchestra.run.smithi165.stdout: 2024-09-17T11:49:47.552 INFO:teuthology.orchestra.run.smithi165.stdout:{"epoch":1,"fsid":"a12537de-74ea-11ef-bceb-c7b262605968","modified":"2024-09-17T11:48:00.407559Z","created":"2024-09-17T11:48:00.407559Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi045","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:3300","nonce":0},{"type":"v1","addr":"172.21.15.45:6789","nonce":0}]},"addr":"172.21.15.45:6789/0","public_addr":"172.21.15.45:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T11:49:47.554 INFO:teuthology.orchestra.run.smithi165.stderr:dumped monmap epoch 1 2024-09-17T11:49:48.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:47 smithi045 ceph-mon[23737]: from='client.? 172.21.15.165:0/210640726' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T11:49:49.103 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T11:49:49.103 DEBUG:teuthology.orchestra.run.smithi165:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T11:49:50.952 INFO:teuthology.orchestra.run.smithi165.stdout: 2024-09-17T11:49:50.952 INFO:teuthology.orchestra.run.smithi165.stdout:{"epoch":1,"fsid":"a12537de-74ea-11ef-bceb-c7b262605968","modified":"2024-09-17T11:48:00.407559Z","created":"2024-09-17T11:48:00.407559Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi045","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:3300","nonce":0},{"type":"v1","addr":"172.21.15.45:6789","nonce":0}]},"addr":"172.21.15.45:6789/0","public_addr":"172.21.15.45:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T11:49:50.954 INFO:teuthology.orchestra.run.smithi165.stderr:dumped monmap epoch 1 2024-09-17T11:49:51.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:51 smithi045 ceph-mon[23737]: Active manager daemon smithi045.jtwyrp restarted 2024-09-17T11:49:51.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:51 smithi045 ceph-mon[23737]: Activating manager daemon smithi045.jtwyrp 2024-09-17T11:49:51.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:51 smithi045 ceph-mon[23737]: osdmap e5: 0 total, 0 up, 0 in 2024-09-17T11:49:51.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:51 smithi045 ceph-mon[23737]: mgrmap e15: smithi045.jtwyrp(active, starting, since 0.0524136s) 2024-09-17T11:49:51.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi045"}]: dispatch 2024-09-17T11:49:51.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mgr metadata", "who": "smithi045.jtwyrp", "id": "smithi045.jtwyrp"}]: dispatch 2024-09-17T11:49:51.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T11:49:51.841 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T11:49:51.841 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T11:49:51.841 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:51 smithi045 ceph-mon[23737]: Manager daemon smithi045.jtwyrp is now available 2024-09-17T11:49:51.841 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:49:51.841 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:49:51.841 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:49:51.841 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:49:51.841 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:51.841 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:49:51.841 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:49:51.841 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:51 smithi045 ceph-mon[23737]: from='client.? 172.21.15.165:0/699293123' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T11:49:51.842 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:52.555 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T11:49:52.555 DEBUG:teuthology.orchestra.run.smithi165:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T11:49:52.975 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:52 smithi045 ceph-mon[23737]: mgrmap e16: smithi045.jtwyrp(active, since 1.05493s) 2024-09-17T11:49:52.975 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:52 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:52.975 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:52 smithi045 ceph-mon[23737]: [17/Sep/2024:11:49:52] ENGINE Bus STARTING 2024-09-17T11:49:52.975 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:52 smithi045 ceph-mon[23737]: [17/Sep/2024:11:49:52] ENGINE Serving on https://172.21.15.45:7150 2024-09-17T11:49:52.975 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:52 smithi045 ceph-mon[23737]: [17/Sep/2024:11:49:52] ENGINE Bus STARTED 2024-09-17T11:49:52.975 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:52 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:53.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:53 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:53.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:53 smithi045 ceph-mon[23737]: mgrmap e17: smithi045.jtwyrp(active, since 2s) 2024-09-17T11:49:55.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:54 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:55.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:54 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:55.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:54 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd/host:smithi045", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:49:55.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:54 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:55.703 INFO:teuthology.orchestra.run.smithi165.stdout: 2024-09-17T11:49:55.703 INFO:teuthology.orchestra.run.smithi165.stdout:{"epoch":1,"fsid":"a12537de-74ea-11ef-bceb-c7b262605968","modified":"2024-09-17T11:48:00.407559Z","created":"2024-09-17T11:48:00.407559Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi045","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:3300","nonce":0},{"type":"v1","addr":"172.21.15.45:6789","nonce":0}]},"addr":"172.21.15.45:6789/0","public_addr":"172.21.15.45:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T11:49:55.705 INFO:teuthology.orchestra.run.smithi165.stderr:dumped monmap epoch 1 2024-09-17T11:49:56.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:55 smithi045 ceph-mon[23737]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-17T11:49:56.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:55 smithi045 ceph-mon[23737]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2024-09-17T11:49:56.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:55 smithi045 ceph-mon[23737]: from='client.? 172.21.15.165:0/2478575393' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T11:49:57.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:57 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:57.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:57 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:57.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:57 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd/host:smithi165", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:49:57.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:57 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:49:57.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:57 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi165", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T11:49:57.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:57 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi165", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-17T11:49:57.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:57 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:49:57.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:49:57 smithi045 ceph-mon[23737]: Deploying daemon crash.smithi165 on smithi165 2024-09-17T11:49:58.012 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T11:49:58.013 DEBUG:teuthology.orchestra.run.smithi165:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T11:50:00.082 INFO:teuthology.orchestra.run.smithi165.stdout: 2024-09-17T11:50:00.082 INFO:teuthology.orchestra.run.smithi165.stdout:{"epoch":1,"fsid":"a12537de-74ea-11ef-bceb-c7b262605968","modified":"2024-09-17T11:48:00.407559Z","created":"2024-09-17T11:48:00.407559Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi045","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:3300","nonce":0},{"type":"v1","addr":"172.21.15.45:6789","nonce":0}]},"addr":"172.21.15.45:6789/0","public_addr":"172.21.15.45:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T11:50:00.083 INFO:teuthology.orchestra.run.smithi165.stderr:dumped monmap epoch 1 2024-09-17T11:50:00.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:00 smithi045 ceph-mon[23737]: from='client.? 172.21.15.165:0/3191545662' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T11:50:01.771 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T11:50:01.771 DEBUG:teuthology.orchestra.run.smithi165:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T11:50:02.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:01 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:02.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:01 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi165.ykdszc", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T11:50:02.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:01 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi165.ykdszc", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-17T11:50:02.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:01 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T11:50:02.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:01 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:50:02.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:01 smithi045 ceph-mon[23737]: Deploying daemon mgr.smithi165.ykdszc on smithi165 2024-09-17T11:50:04.047 INFO:teuthology.orchestra.run.smithi165.stdout: 2024-09-17T11:50:04.048 INFO:teuthology.orchestra.run.smithi165.stdout:{"epoch":1,"fsid":"a12537de-74ea-11ef-bceb-c7b262605968","modified":"2024-09-17T11:48:00.407559Z","created":"2024-09-17T11:48:00.407559Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi045","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:3300","nonce":0},{"type":"v1","addr":"172.21.15.45:6789","nonce":0}]},"addr":"172.21.15.45:6789/0","public_addr":"172.21.15.45:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T11:50:04.049 INFO:teuthology.orchestra.run.smithi165.stderr:dumped monmap epoch 1 2024-09-17T11:50:04.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:04 smithi045 ceph-mon[23737]: from='client.? 172.21.15.165:0/267719752' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T11:50:05.578 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T11:50:05.578 DEBUG:teuthology.orchestra.run.smithi165:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T11:50:06.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:05 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:06.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:05 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T11:50:06.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:05 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:50:06.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:05 smithi045 ceph-mon[23737]: Deploying daemon mon.smithi165 on smithi165 2024-09-17T11:50:06.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:05 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:08.885 INFO:teuthology.orchestra.run.smithi165.stdout: 2024-09-17T11:50:08.885 INFO:teuthology.orchestra.run.smithi165.stdout:{"epoch":1,"fsid":"a12537de-74ea-11ef-bceb-c7b262605968","modified":"2024-09-17T11:48:00.407559Z","created":"2024-09-17T11:48:00.407559Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi045","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:3300","nonce":0},{"type":"v1","addr":"172.21.15.45:6789","nonce":0}]},"addr":"172.21.15.45:6789/0","public_addr":"172.21.15.45:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T11:50:08.887 INFO:teuthology.orchestra.run.smithi165.stderr:dumped monmap epoch 1 2024-09-17T11:50:09.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:08 smithi045 ceph-mon[23737]: from='client.? 172.21.15.165:0/3535579808' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T11:50:10.692 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T11:50:10.693 DEBUG:teuthology.orchestra.run.smithi165:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T11:50:14.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:14 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi045"}]: dispatch 2024-09-17T11:50:14.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:14 smithi045 ceph-mon[23737]: mon.smithi045 calling monitor election 2024-09-17T11:50:14.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:14 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi165"}]: dispatch 2024-09-17T11:50:14.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:14 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi165"}]: dispatch 2024-09-17T11:50:14.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:14 smithi045 ceph-mon[23737]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:14.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:14 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi165"}]: dispatch 2024-09-17T11:50:14.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:14 smithi045 ceph-mon[23737]: mon.smithi165 calling monitor election 2024-09-17T11:50:14.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:14 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.165:0/3426638818' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi165.ykdszc/crt"}]: dispatch 2024-09-17T11:50:14.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:14 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi165"}]: dispatch 2024-09-17T11:50:14.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:14 smithi045 ceph-mon[23737]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:14.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:14 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi165"}]: dispatch 2024-09-17T11:50:14.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:14 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi165"}]: dispatch 2024-09-17T11:50:14.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:14 smithi045 ceph-mon[23737]: mon.smithi045 is new leader, mons smithi045,smithi165 in quorum (ranks 0,1) 2024-09-17T11:50:14.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:14 smithi045 ceph-mon[23737]: monmap e2: 2 mons at {smithi045=[v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0],smithi165=[v2:172.21.15.165:3300/0,v1:172.21.15.165:6789/0]} 2024-09-17T11:50:14.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:14 smithi045 ceph-mon[23737]: fsmap 2024-09-17T11:50:14.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:14 smithi045 ceph-mon[23737]: osdmap e5: 0 total, 0 up, 0 in 2024-09-17T11:50:14.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:14 smithi045 ceph-mon[23737]: mgrmap e17: smithi045.jtwyrp(active, since 23s) 2024-09-17T11:50:14.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:14 smithi045 ceph-mon[23737]: Standby manager daemon smithi165.ykdszc started 2024-09-17T11:50:14.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:14 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.165:0/3426638818' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T11:50:14.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:14 smithi045 ceph-mon[23737]: overall HEALTH_OK 2024-09-17T11:50:14.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:14 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.165:0/3426638818' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi165.ykdszc/key"}]: dispatch 2024-09-17T11:50:14.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:14 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:14.342 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:14 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.165:0/3426638818' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T11:50:15.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:15 smithi045 ceph-mon[23737]: Deploying daemon node-exporter.smithi165 on smithi165 2024-09-17T11:50:15.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:15 smithi045 ceph-mon[23737]: mgrmap e18: smithi045.jtwyrp(active, since 23s), standbys: smithi165.ykdszc 2024-09-17T11:50:15.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:15 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mgr metadata", "who": "smithi165.ykdszc", "id": "smithi165.ykdszc"}]: dispatch 2024-09-17T11:50:15.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:15 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi165"}]: dispatch 2024-09-17T11:50:15.362 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:15 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi165"}]: dispatch 2024-09-17T11:50:15.525 INFO:teuthology.orchestra.run.smithi165.stdout: 2024-09-17T11:50:15.525 INFO:teuthology.orchestra.run.smithi165.stdout:{"epoch":2,"fsid":"a12537de-74ea-11ef-bceb-c7b262605968","modified":"2024-09-17T11:50:08.996728Z","created":"2024-09-17T11:48:00.407559Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi045","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:3300","nonce":0},{"type":"v1","addr":"172.21.15.45:6789","nonce":0}]},"addr":"172.21.15.45:6789/0","public_addr":"172.21.15.45:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi165","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:3300","nonce":0},{"type":"v1","addr":"172.21.15.165:6789","nonce":0}]},"addr":"172.21.15.165:6789/0","public_addr":"172.21.15.165:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-17T11:50:15.527 INFO:teuthology.orchestra.run.smithi165.stderr:dumped monmap epoch 2 2024-09-17T11:50:16.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:16 smithi045 ceph-mon[23737]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:16.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:16 smithi045 ceph-mon[23737]: from='client.? 172.21.15.165:0/3489675384' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T11:50:16.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:16 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:16.429 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:16 smithi165 ceph-mon[25702]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:16.430 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:16 smithi165 ceph-mon[25702]: from='client.? 172.21.15.165:0/3489675384' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T11:50:16.430 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:16 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:16.451 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-17T11:50:16.452 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-17T11:50:18.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:18 smithi045 ceph-mon[23737]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:18.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:18 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:18.377 INFO:teuthology.orchestra.run.smithi045.stdout:# minimal ceph.conf for a12537de-74ea-11ef-bceb-c7b262605968 2024-09-17T11:50:18.377 INFO:teuthology.orchestra.run.smithi045.stdout:[global] 2024-09-17T11:50:18.377 INFO:teuthology.orchestra.run.smithi045.stdout: fsid = a12537de-74ea-11ef-bceb-c7b262605968 2024-09-17T11:50:18.377 INFO:teuthology.orchestra.run.smithi045.stdout: mon_host = [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] [v2:172.21.15.165:3300/0,v1:172.21.15.165:6789/0] 2024-09-17T11:50:18.496 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:18 smithi165 ceph-mon[25702]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:18.497 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:18 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:18.925 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-17T11:50:18.925 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2024-09-17T11:50:18.925 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd of=/etc/ceph/ceph.conf 2024-09-17T11:50:18.953 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2024-09-17T11:50:18.953 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-17T11:50:19.019 DEBUG:teuthology.orchestra.run.smithi165:> set -ex 2024-09-17T11:50:19.019 DEBUG:teuthology.orchestra.run.smithi165:> sudo dd of=/etc/ceph/ceph.conf 2024-09-17T11:50:19.053 DEBUG:teuthology.orchestra.run.smithi165:> set -ex 2024-09-17T11:50:19.053 DEBUG:teuthology.orchestra.run.smithi165:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-17T11:50:19.122 DEBUG:tasks.cephadm:set 0 configs 2024-09-17T11:50:19.122 INFO:tasks.cephadm:Deploying OSDs... 2024-09-17T11:50:19.122 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2024-09-17T11:50:19.122 DEBUG:teuthology.orchestra.run.smithi045:> dd if=/scratch_devs of=/dev/stdout 2024-09-17T11:50:19.139 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-17T11:50:19.139 DEBUG:teuthology.orchestra.run.smithi045:> stat /dev/vg_nvme/lv_1 2024-09-17T11:50:19.196 INFO:teuthology.orchestra.run.smithi045.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-17T11:50:19.196 INFO:teuthology.orchestra.run.smithi045.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T11:50:19.196 INFO:teuthology.orchestra.run.smithi045.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-09-17T11:50:19.196 INFO:teuthology.orchestra.run.smithi045.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T11:50:19.196 INFO:teuthology.orchestra.run.smithi045.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T11:50:19.197 INFO:teuthology.orchestra.run.smithi045.stdout:Access: 2024-09-17 11:49:40.778878037 +0000 2024-09-17T11:50:19.197 INFO:teuthology.orchestra.run.smithi045.stdout:Modify: 2024-09-17 11:47:13.617819981 +0000 2024-09-17T11:50:19.197 INFO:teuthology.orchestra.run.smithi045.stdout:Change: 2024-09-17 11:47:13.617819981 +0000 2024-09-17T11:50:19.197 INFO:teuthology.orchestra.run.smithi045.stdout: Birth: 2024-09-17 11:47:13.617819981 +0000 2024-09-17T11:50:19.197 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-17T11:50:19.259 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:19 smithi045 ceph-mon[23737]: Deploying daemon prometheus.smithi045 on smithi045 2024-09-17T11:50:19.259 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:19 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/1985890463' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:50:19.260 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records in 2024-09-17T11:50:19.260 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records out 2024-09-17T11:50:19.260 INFO:teuthology.orchestra.run.smithi045.stderr:512 bytes copied, 9.4309e-05 s, 5.4 MB/s 2024-09-17T11:50:19.262 DEBUG:teuthology.orchestra.run.smithi045:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-17T11:50:19.320 DEBUG:teuthology.orchestra.run.smithi045:> stat /dev/vg_nvme/lv_2 2024-09-17T11:50:19.376 INFO:teuthology.orchestra.run.smithi045.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-17T11:50:19.376 INFO:teuthology.orchestra.run.smithi045.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T11:50:19.376 INFO:teuthology.orchestra.run.smithi045.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-09-17T11:50:19.376 INFO:teuthology.orchestra.run.smithi045.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T11:50:19.376 INFO:teuthology.orchestra.run.smithi045.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T11:50:19.376 INFO:teuthology.orchestra.run.smithi045.stdout:Access: 2024-09-17 11:49:40.779878022 +0000 2024-09-17T11:50:19.376 INFO:teuthology.orchestra.run.smithi045.stdout:Modify: 2024-09-17 11:47:13.945814977 +0000 2024-09-17T11:50:19.376 INFO:teuthology.orchestra.run.smithi045.stdout:Change: 2024-09-17 11:47:13.945814977 +0000 2024-09-17T11:50:19.377 INFO:teuthology.orchestra.run.smithi045.stdout: Birth: 2024-09-17 11:47:13.945814977 +0000 2024-09-17T11:50:19.377 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-17T11:50:19.442 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records in 2024-09-17T11:50:19.442 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records out 2024-09-17T11:50:19.442 INFO:teuthology.orchestra.run.smithi045.stderr:512 bytes copied, 0.000104198 s, 4.9 MB/s 2024-09-17T11:50:19.443 DEBUG:teuthology.orchestra.run.smithi045:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-17T11:50:19.501 DEBUG:teuthology.orchestra.run.smithi045:> stat /dev/vg_nvme/lv_3 2024-09-17T11:50:19.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:19 smithi165 ceph-mon[25702]: Deploying daemon prometheus.smithi045 on smithi045 2024-09-17T11:50:19.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:19 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/1985890463' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:50:19.556 INFO:teuthology.orchestra.run.smithi045.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-17T11:50:19.556 INFO:teuthology.orchestra.run.smithi045.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T11:50:19.556 INFO:teuthology.orchestra.run.smithi045.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-09-17T11:50:19.556 INFO:teuthology.orchestra.run.smithi045.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T11:50:19.556 INFO:teuthology.orchestra.run.smithi045.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T11:50:19.556 INFO:teuthology.orchestra.run.smithi045.stdout:Access: 2024-09-17 11:49:40.779878022 +0000 2024-09-17T11:50:19.556 INFO:teuthology.orchestra.run.smithi045.stdout:Modify: 2024-09-17 11:47:14.262810141 +0000 2024-09-17T11:50:19.556 INFO:teuthology.orchestra.run.smithi045.stdout:Change: 2024-09-17 11:47:14.262810141 +0000 2024-09-17T11:50:19.557 INFO:teuthology.orchestra.run.smithi045.stdout: Birth: 2024-09-17 11:47:14.262810141 +0000 2024-09-17T11:50:19.557 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-17T11:50:19.620 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records in 2024-09-17T11:50:19.620 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records out 2024-09-17T11:50:19.620 INFO:teuthology.orchestra.run.smithi045.stderr:512 bytes copied, 8.8988e-05 s, 5.8 MB/s 2024-09-17T11:50:19.622 DEBUG:teuthology.orchestra.run.smithi045:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-17T11:50:19.678 DEBUG:teuthology.orchestra.run.smithi045:> stat /dev/vg_nvme/lv_4 2024-09-17T11:50:19.735 INFO:teuthology.orchestra.run.smithi045.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-17T11:50:19.735 INFO:teuthology.orchestra.run.smithi045.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T11:50:19.735 INFO:teuthology.orchestra.run.smithi045.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-09-17T11:50:19.735 INFO:teuthology.orchestra.run.smithi045.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T11:50:19.735 INFO:teuthology.orchestra.run.smithi045.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T11:50:19.735 INFO:teuthology.orchestra.run.smithi045.stdout:Access: 2024-09-17 11:49:40.779878022 +0000 2024-09-17T11:50:19.735 INFO:teuthology.orchestra.run.smithi045.stdout:Modify: 2024-09-17 11:47:14.582805259 +0000 2024-09-17T11:50:19.735 INFO:teuthology.orchestra.run.smithi045.stdout:Change: 2024-09-17 11:47:14.582805259 +0000 2024-09-17T11:50:19.735 INFO:teuthology.orchestra.run.smithi045.stdout: Birth: 2024-09-17 11:47:14.582805259 +0000 2024-09-17T11:50:19.736 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-17T11:50:19.799 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records in 2024-09-17T11:50:19.800 INFO:teuthology.orchestra.run.smithi045.stderr:1+0 records out 2024-09-17T11:50:19.800 INFO:teuthology.orchestra.run.smithi045.stderr:512 bytes copied, 9.1868e-05 s, 5.6 MB/s 2024-09-17T11:50:19.801 DEBUG:teuthology.orchestra.run.smithi045:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-17T11:50:19.857 DEBUG:teuthology.orchestra.run.smithi165:> set -ex 2024-09-17T11:50:19.857 DEBUG:teuthology.orchestra.run.smithi165:> dd if=/scratch_devs of=/dev/stdout 2024-09-17T11:50:19.876 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-17T11:50:19.876 DEBUG:teuthology.orchestra.run.smithi165:> stat /dev/vg_nvme/lv_1 2024-09-17T11:50:19.935 INFO:teuthology.orchestra.run.smithi165.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-17T11:50:19.935 INFO:teuthology.orchestra.run.smithi165.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T11:50:19.935 INFO:teuthology.orchestra.run.smithi165.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-09-17T11:50:19.935 INFO:teuthology.orchestra.run.smithi165.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T11:50:19.935 INFO:teuthology.orchestra.run.smithi165.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T11:50:19.935 INFO:teuthology.orchestra.run.smithi165.stdout:Access: 2024-09-17 11:50:14.594361342 +0000 2024-09-17T11:50:19.935 INFO:teuthology.orchestra.run.smithi165.stdout:Modify: 2024-09-17 11:47:13.618541388 +0000 2024-09-17T11:50:19.935 INFO:teuthology.orchestra.run.smithi165.stdout:Change: 2024-09-17 11:47:13.618541388 +0000 2024-09-17T11:50:19.935 INFO:teuthology.orchestra.run.smithi165.stdout: Birth: 2024-09-17 11:47:13.618541388 +0000 2024-09-17T11:50:19.936 DEBUG:teuthology.orchestra.run.smithi165:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-17T11:50:20.002 INFO:teuthology.orchestra.run.smithi165.stderr:1+0 records in 2024-09-17T11:50:20.002 INFO:teuthology.orchestra.run.smithi165.stderr:1+0 records out 2024-09-17T11:50:20.002 INFO:teuthology.orchestra.run.smithi165.stderr:512 bytes copied, 0.000265609 s, 1.9 MB/s 2024-09-17T11:50:20.003 DEBUG:teuthology.orchestra.run.smithi165:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-17T11:50:20.062 DEBUG:teuthology.orchestra.run.smithi165:> stat /dev/vg_nvme/lv_2 2024-09-17T11:50:20.119 INFO:teuthology.orchestra.run.smithi165.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-17T11:50:20.119 INFO:teuthology.orchestra.run.smithi165.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T11:50:20.119 INFO:teuthology.orchestra.run.smithi165.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-09-17T11:50:20.119 INFO:teuthology.orchestra.run.smithi165.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T11:50:20.119 INFO:teuthology.orchestra.run.smithi165.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T11:50:20.119 INFO:teuthology.orchestra.run.smithi165.stdout:Access: 2024-09-17 11:50:14.595361326 +0000 2024-09-17T11:50:20.119 INFO:teuthology.orchestra.run.smithi165.stdout:Modify: 2024-09-17 11:47:13.943536334 +0000 2024-09-17T11:50:20.119 INFO:teuthology.orchestra.run.smithi165.stdout:Change: 2024-09-17 11:47:13.943536334 +0000 2024-09-17T11:50:20.119 INFO:teuthology.orchestra.run.smithi165.stdout: Birth: 2024-09-17 11:47:13.943536334 +0000 2024-09-17T11:50:20.120 DEBUG:teuthology.orchestra.run.smithi165:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-17T11:50:20.184 INFO:teuthology.orchestra.run.smithi165.stderr:1+0 records in 2024-09-17T11:50:20.184 INFO:teuthology.orchestra.run.smithi165.stderr:1+0 records out 2024-09-17T11:50:20.184 INFO:teuthology.orchestra.run.smithi165.stderr:512 bytes copied, 0.000228819 s, 2.2 MB/s 2024-09-17T11:50:20.187 DEBUG:teuthology.orchestra.run.smithi165:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-17T11:50:20.244 DEBUG:teuthology.orchestra.run.smithi165:> stat /dev/vg_nvme/lv_3 2024-09-17T11:50:20.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:20 smithi165 ceph-mon[25702]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:20.281 INFO:teuthology.orchestra.run.smithi165.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-17T11:50:20.281 INFO:teuthology.orchestra.run.smithi165.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T11:50:20.282 INFO:teuthology.orchestra.run.smithi165.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-09-17T11:50:20.282 INFO:teuthology.orchestra.run.smithi165.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T11:50:20.282 INFO:teuthology.orchestra.run.smithi165.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T11:50:20.282 INFO:teuthology.orchestra.run.smithi165.stdout:Access: 2024-09-17 11:50:14.595361326 +0000 2024-09-17T11:50:20.282 INFO:teuthology.orchestra.run.smithi165.stdout:Modify: 2024-09-17 11:47:14.266531311 +0000 2024-09-17T11:50:20.282 INFO:teuthology.orchestra.run.smithi165.stdout:Change: 2024-09-17 11:47:14.266531311 +0000 2024-09-17T11:50:20.282 INFO:teuthology.orchestra.run.smithi165.stdout: Birth: 2024-09-17 11:47:14.266531311 +0000 2024-09-17T11:50:20.282 DEBUG:teuthology.orchestra.run.smithi165:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-17T11:50:20.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:20 smithi045 ceph-mon[23737]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:20.345 INFO:teuthology.orchestra.run.smithi165.stderr:1+0 records in 2024-09-17T11:50:20.346 INFO:teuthology.orchestra.run.smithi165.stderr:1+0 records out 2024-09-17T11:50:20.346 INFO:teuthology.orchestra.run.smithi165.stderr:512 bytes copied, 0.000240788 s, 2.1 MB/s 2024-09-17T11:50:20.347 DEBUG:teuthology.orchestra.run.smithi165:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-17T11:50:20.404 DEBUG:teuthology.orchestra.run.smithi165:> stat /dev/vg_nvme/lv_4 2024-09-17T11:50:20.459 INFO:teuthology.orchestra.run.smithi165.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-17T11:50:20.459 INFO:teuthology.orchestra.run.smithi165.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T11:50:20.460 INFO:teuthology.orchestra.run.smithi165.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-17T11:50:20.460 INFO:teuthology.orchestra.run.smithi165.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T11:50:20.460 INFO:teuthology.orchestra.run.smithi165.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T11:50:20.460 INFO:teuthology.orchestra.run.smithi165.stdout:Access: 2024-09-17 11:50:14.596361310 +0000 2024-09-17T11:50:20.460 INFO:teuthology.orchestra.run.smithi165.stdout:Modify: 2024-09-17 11:47:14.584526366 +0000 2024-09-17T11:50:20.460 INFO:teuthology.orchestra.run.smithi165.stdout:Change: 2024-09-17 11:47:14.584526366 +0000 2024-09-17T11:50:20.460 INFO:teuthology.orchestra.run.smithi165.stdout: Birth: 2024-09-17 11:47:14.584526366 +0000 2024-09-17T11:50:20.460 DEBUG:teuthology.orchestra.run.smithi165:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-17T11:50:20.522 INFO:teuthology.orchestra.run.smithi165.stderr:1+0 records in 2024-09-17T11:50:20.522 INFO:teuthology.orchestra.run.smithi165.stderr:1+0 records out 2024-09-17T11:50:20.523 INFO:teuthology.orchestra.run.smithi165.stderr:512 bytes copied, 0.0001981 s, 2.6 MB/s 2024-09-17T11:50:20.524 DEBUG:teuthology.orchestra.run.smithi165:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-17T11:50:20.580 INFO:tasks.cephadm:Deploying osd.0 on smithi045 with /dev/vg_nvme/lv_4... 2024-09-17T11:50:20.580 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-17T11:50:22.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:21 smithi165 ceph-mon[25702]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:22.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:21 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:22.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:21 smithi045 ceph-mon[23737]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:22.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:21 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:23.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:23 smithi045 ceph-mon[23737]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:24.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:23 smithi165 ceph-mon[25702]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:26.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:25 smithi165 ceph-mon[25702]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:26.069 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:25 smithi045 ceph-mon[23737]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:28.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:27 smithi165 ceph-mon[25702]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:28.056 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:27 smithi045 ceph-mon[23737]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:29.960 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:29 smithi045 ceph-mon[23737]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:29.960 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:29 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:29.960 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:29 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:50:29.960 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:29 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:50:29.960 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:29 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:50:30.173 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:29 smithi165 ceph-mon[25702]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:30.173 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:29 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:30.173 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:29 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:50:30.173 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:29 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:50:30.173 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:29 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:50:30.650 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:50:30.672 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi045:vg_nvme/lv_4 2024-09-17T11:50:31.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:31 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:31.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:31 smithi045 ceph-mon[23737]: Updating smithi165:/etc/ceph/ceph.conf 2024-09-17T11:50:31.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:31 smithi045 ceph-mon[23737]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:31.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:31 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:31.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:31 smithi045 ceph-mon[23737]: Updating smithi165:/etc/ceph/ceph.client.admin.keyring 2024-09-17T11:50:31.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:31 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:32.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:31 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:32.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:31 smithi165 ceph-mon[25702]: Updating smithi165:/etc/ceph/ceph.conf 2024-09-17T11:50:32.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:31 smithi165 ceph-mon[25702]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:32.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:31 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:32.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:31 smithi165 ceph-mon[25702]: Updating smithi165:/etc/ceph/ceph.client.admin.keyring 2024-09-17T11:50:32.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:31 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:33.796 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:33 smithi045 ceph-mon[23737]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:34.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:33 smithi165 ceph-mon[25702]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:34.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:34 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T11:50:34.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:34 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T11:50:34.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:34 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:50:35.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:34 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T11:50:35.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:34 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T11:50:35.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:34 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:50:35.924 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:35 smithi045 ceph-mon[23737]: from='client.14232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:50:35.924 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:35 smithi045 ceph-mon[23737]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:36.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:35 smithi165 ceph-mon[25702]: from='client.14232 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:50:36.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:35 smithi165 ceph-mon[25702]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:37.886 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:37 smithi045 ceph-mon[23737]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:38.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:37 smithi165 ceph-mon[25702]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:39.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:38 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:39.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:38 smithi045 ceph-mon[23737]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-17T11:50:39.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:38 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/1961254484' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e288efb6-6c20-40b0-95fd-75180fb6f1c8"}]: dispatch 2024-09-17T11:50:39.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:38 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/1961254484' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e288efb6-6c20-40b0-95fd-75180fb6f1c8"}]': finished 2024-09-17T11:50:39.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:38 smithi045 ceph-mon[23737]: osdmap e6: 1 total, 0 up, 1 in 2024-09-17T11:50:39.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:38 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T11:50:39.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:38 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:39.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:38 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:39.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:38 smithi045 ceph-mon[23737]: Reconfiguring mon.smithi045 (unknown last config time)... 2024-09-17T11:50:39.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:38 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T11:50:39.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:38 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T11:50:39.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:38 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:50:39.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:38 smithi045 ceph-mon[23737]: Reconfiguring daemon mon.smithi045 on smithi045 2024-09-17T11:50:39.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:38 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/2543052094' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T11:50:39.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:38 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:39.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:38 smithi165 ceph-mon[25702]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-17T11:50:39.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:38 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/1961254484' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e288efb6-6c20-40b0-95fd-75180fb6f1c8"}]: dispatch 2024-09-17T11:50:39.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:38 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/1961254484' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e288efb6-6c20-40b0-95fd-75180fb6f1c8"}]': finished 2024-09-17T11:50:39.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:38 smithi165 ceph-mon[25702]: osdmap e6: 1 total, 0 up, 1 in 2024-09-17T11:50:39.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:38 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T11:50:39.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:38 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:39.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:38 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:39.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:38 smithi165 ceph-mon[25702]: Reconfiguring mon.smithi045 (unknown last config time)... 2024-09-17T11:50:39.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:38 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T11:50:39.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:38 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T11:50:39.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:38 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:50:39.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:38 smithi165 ceph-mon[25702]: Reconfiguring daemon mon.smithi045 on smithi045 2024-09-17T11:50:39.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:38 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/2543052094' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T11:50:40.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:40 smithi045 ceph-mon[23737]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:40.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:40 smithi165 ceph-mon[25702]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:42.317 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:42 smithi165 ceph-mon[25702]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:42.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:42 smithi045 ceph-mon[23737]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:43.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:43 smithi165 ceph-mon[25702]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:43.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:43.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:43 smithi165 ceph-mon[25702]: Reconfiguring mgr.smithi045.jtwyrp (unknown last config time)... 2024-09-17T11:50:43.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi045.jtwyrp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T11:50:43.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T11:50:43.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:50:43.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:43 smithi165 ceph-mon[25702]: Reconfiguring daemon mgr.smithi045.jtwyrp on smithi045 2024-09-17T11:50:43.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:43 smithi045 ceph-mon[23737]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:43.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:43.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:43 smithi045 ceph-mon[23737]: Reconfiguring mgr.smithi045.jtwyrp (unknown last config time)... 2024-09-17T11:50:43.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi045.jtwyrp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T11:50:43.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T11:50:43.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:50:43.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:43 smithi045 ceph-mon[23737]: Reconfiguring daemon mgr.smithi045.jtwyrp on smithi045 2024-09-17T11:50:46.080 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:45 smithi045 ceph-mon[23737]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:46.080 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:45 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:46.080 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:45 smithi045 ceph-mon[23737]: Reconfiguring alertmanager.smithi045 (dependencies changed)... 2024-09-17T11:50:46.080 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:45 smithi045 ceph-mon[23737]: Reconfiguring daemon alertmanager.smithi045 on smithi045 2024-09-17T11:50:46.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:45 smithi165 ceph-mon[25702]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:46.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:45 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:46.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:45 smithi165 ceph-mon[25702]: Reconfiguring alertmanager.smithi045 (dependencies changed)... 2024-09-17T11:50:46.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:45 smithi165 ceph-mon[25702]: Reconfiguring daemon alertmanager.smithi045 on smithi045 2024-09-17T11:50:47.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:46 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T11:50:47.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:46 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:50:47.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:46 smithi045 ceph-mon[23737]: Deploying daemon osd.0 on smithi045 2024-09-17T11:50:47.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:46 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T11:50:47.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:46 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:50:47.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:46 smithi165 ceph-mon[25702]: Deploying daemon osd.0 on smithi045 2024-09-17T11:50:48.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:47 smithi045 ceph-mon[23737]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:48.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:47 smithi165 ceph-mon[25702]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:50.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:49 smithi045 ceph-mon[23737]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:49 smithi165 ceph-mon[25702]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:50.977 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:50 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:50:50.977 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:50 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:50.978 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:50 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:50:50.978 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:50 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi045", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T11:50:50.978 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:50 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:50:51.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:50 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:50:51.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:50 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:51.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:50 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:50:51.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:50 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi045", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T11:50:51.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:50 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:50:52.193 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:51 smithi045 ceph-mon[23737]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:52.194 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:51 smithi045 ceph-mon[23737]: Reconfiguring crash.smithi045 (monmap changed)... 2024-09-17T11:50:52.194 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:51 smithi045 ceph-mon[23737]: Reconfiguring daemon crash.smithi045 on smithi045 2024-09-17T11:50:52.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:51 smithi165 ceph-mon[25702]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:52.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:51 smithi165 ceph-mon[25702]: Reconfiguring crash.smithi045 (monmap changed)... 2024-09-17T11:50:52.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:51 smithi165 ceph-mon[25702]: Reconfiguring daemon crash.smithi045 on smithi045 2024-09-17T11:50:54.189 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:53 smithi045 ceph-mon[23737]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:54.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:53 smithi165 ceph-mon[25702]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:56.170 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:55 smithi045 ceph-mon[23737]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:56.170 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:55 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:56.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:55 smithi165 ceph-mon[25702]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:56.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:55 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:57.959 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:57 smithi045 ceph-mon[23737]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:57.960 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:57 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:58.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:57 smithi165 ceph-mon[25702]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:50:58.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:57 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:50:59.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:58 smithi165 ceph-mon[25702]: Reconfiguring grafana.smithi045 (dependencies changed)... 2024-09-17T11:50:59.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:58 smithi165 ceph-mon[25702]: Reconfiguring daemon grafana.smithi045 on smithi045 2024-09-17T11:50:59.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:58 smithi045 ceph-mon[23737]: Reconfiguring grafana.smithi045 (dependencies changed)... 2024-09-17T11:50:59.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:58 smithi045 ceph-mon[23737]: Reconfiguring daemon grafana.smithi045 on smithi045 2024-09-17T11:50:59.698 INFO:teuthology.orchestra.run.smithi045.stdout:Created osd(s) 0 on host 'smithi045' 2024-09-17T11:51:00.256 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:59 smithi045 ceph-mon[23737]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:51:00.256 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:59 smithi045 ceph-mon[23737]: from='osd.0 [v2:172.21.15.45:6802/3351925855,v1:172.21.15.45:6803/3351925855]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T11:51:00.256 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:50:59 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:00.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:59 smithi165 ceph-mon[25702]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:51:00.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:59 smithi165 ceph-mon[25702]: from='osd.0 [v2:172.21.15.45:6802/3351925855,v1:172.21.15.45:6803/3351925855]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T11:51:00.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:50:59 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:00.286 DEBUG:teuthology.orchestra.run.smithi045:osd.0> sudo journalctl -f -n 0 -u ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.0.service 2024-09-17T11:51:00.289 INFO:tasks.cephadm:Deploying osd.1 on smithi045 with /dev/vg_nvme/lv_3... 2024-09-17T11:51:00.289 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-17T11:51:01.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:00 smithi165 ceph-mon[25702]: from='osd.0 [v2:172.21.15.45:6802/3351925855,v1:172.21.15.45:6803/3351925855]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T11:51:01.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:00 smithi165 ceph-mon[25702]: osdmap e7: 1 total, 0 up, 1 in 2024-09-17T11:51:01.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:00 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T11:51:01.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:00 smithi165 ceph-mon[25702]: from='osd.0 [v2:172.21.15.45:6802/3351925855,v1:172.21.15.45:6803/3351925855]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-17T11:51:01.277 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:00 smithi045 ceph-mon[23737]: from='osd.0 [v2:172.21.15.45:6802/3351925855,v1:172.21.15.45:6803/3351925855]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T11:51:01.277 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:00 smithi045 ceph-mon[23737]: osdmap e7: 1 total, 0 up, 1 in 2024-09-17T11:51:01.277 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:00 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T11:51:01.277 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:00 smithi045 ceph-mon[23737]: from='osd.0 [v2:172.21.15.45:6802/3351925855,v1:172.21.15.45:6803/3351925855]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-17T11:51:01.277 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 11:51:00 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0[34164]: 2024-09-17T11:51:00.936+0000 7faab5286700 -1 osd.0 0 waiting for initial osdmap 2024-09-17T11:51:01.277 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 11:51:00 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0[34164]: 2024-09-17T11:51:00.940+0000 7faaae41a700 -1 osd.0 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T11:51:02.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:01 smithi045 ceph-mon[23737]: purged_snaps scrub starts 2024-09-17T11:51:02.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:01 smithi045 ceph-mon[23737]: purged_snaps scrub ok 2024-09-17T11:51:02.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:01 smithi045 ceph-mon[23737]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:51:02.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:01 smithi045 ceph-mon[23737]: from='osd.0 [v2:172.21.15.45:6802/3351925855,v1:172.21.15.45:6803/3351925855]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2024-09-17T11:51:02.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:01 smithi045 ceph-mon[23737]: osdmap e8: 1 total, 0 up, 1 in 2024-09-17T11:51:02.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:01 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T11:51:02.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:01 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T11:51:02.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:01 smithi165 ceph-mon[25702]: purged_snaps scrub starts 2024-09-17T11:51:02.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:01 smithi165 ceph-mon[25702]: purged_snaps scrub ok 2024-09-17T11:51:02.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:01 smithi165 ceph-mon[25702]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:51:02.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:01 smithi165 ceph-mon[25702]: from='osd.0 [v2:172.21.15.45:6802/3351925855,v1:172.21.15.45:6803/3351925855]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2024-09-17T11:51:02.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:01 smithi165 ceph-mon[25702]: osdmap e8: 1 total, 0 up, 1 in 2024-09-17T11:51:02.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:01 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T11:51:02.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:01 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T11:51:03.076 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:02 smithi045 ceph-mon[23737]: osd.0 [v2:172.21.15.45:6802/3351925855,v1:172.21.15.45:6803/3351925855] boot 2024-09-17T11:51:03.076 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:02 smithi045 ceph-mon[23737]: osdmap e9: 1 total, 1 up, 1 in 2024-09-17T11:51:03.077 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:02 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T11:51:03.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:02 smithi165 ceph-mon[25702]: osd.0 [v2:172.21.15.45:6802/3351925855,v1:172.21.15.45:6803/3351925855] boot 2024-09-17T11:51:03.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:02 smithi165 ceph-mon[25702]: osdmap e9: 1 total, 1 up, 1 in 2024-09-17T11:51:03.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:02 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T11:51:04.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:03 smithi165 ceph-mon[25702]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:51:04.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:03 smithi165 ceph-mon[25702]: osdmap e10: 1 total, 1 up, 1 in 2024-09-17T11:51:04.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:03 smithi045 ceph-mon[23737]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T11:51:04.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:03 smithi045 ceph-mon[23737]: osdmap e10: 1 total, 1 up, 1 in 2024-09-17T11:51:06.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:06 smithi045 ceph-mon[23737]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:06.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:06 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:06.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:06 smithi045 ceph-mon[23737]: Reconfiguring crash.smithi165 (monmap changed)... 2024-09-17T11:51:06.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:06 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi165", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T11:51:06.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:06 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:51:06.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:06 smithi045 ceph-mon[23737]: Reconfiguring daemon crash.smithi165 on smithi165 2024-09-17T11:51:06.310 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:06 smithi165 ceph-mon[25702]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:06.310 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:06 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:06.310 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:06 smithi165 ceph-mon[25702]: Reconfiguring crash.smithi165 (monmap changed)... 2024-09-17T11:51:06.310 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:06 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi165", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T11:51:06.311 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:06 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:51:06.311 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:06 smithi165 ceph-mon[25702]: Reconfiguring daemon crash.smithi165 on smithi165 2024-09-17T11:51:06.466 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:51:06.488 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi045:vg_nvme/lv_3 2024-09-17T11:51:07.467 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:07 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:07.467 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:07 smithi045 ceph-mon[23737]: Reconfiguring mgr.smithi165.ykdszc (monmap changed)... 2024-09-17T11:51:07.467 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:07 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi165.ykdszc", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T11:51:07.467 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:07 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T11:51:07.467 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:07 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:51:07.467 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:07 smithi045 ceph-mon[23737]: Reconfiguring daemon mgr.smithi165.ykdszc on smithi165 2024-09-17T11:51:07.467 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:07 smithi045 ceph-mon[23737]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:07.626 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:07 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:07.626 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:07 smithi165 ceph-mon[25702]: Reconfiguring mgr.smithi165.ykdszc (monmap changed)... 2024-09-17T11:51:07.626 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:07 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi165.ykdszc", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T11:51:07.626 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:07 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T11:51:07.626 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:07 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:51:07.626 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:07 smithi165 ceph-mon[25702]: Reconfiguring daemon mgr.smithi165.ykdszc on smithi165 2024-09-17T11:51:07.626 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:07 smithi165 ceph-mon[25702]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:08.723 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:08 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:08.723 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:08 smithi045 ceph-mon[23737]: Reconfiguring mon.smithi165 (monmap changed)... 2024-09-17T11:51:08.723 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:08 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T11:51:08.723 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:08 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T11:51:08.723 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:08 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:51:08.723 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:08 smithi045 ceph-mon[23737]: Reconfiguring daemon mon.smithi165 on smithi165 2024-09-17T11:51:08.723 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:08 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T11:51:08.723 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:08 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T11:51:08.723 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:08 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:51:08.896 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:08 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:08.897 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:08 smithi165 ceph-mon[25702]: Reconfiguring mon.smithi165 (monmap changed)... 2024-09-17T11:51:08.897 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:08 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T11:51:08.897 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:08 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T11:51:08.897 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:08 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:51:08.897 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:08 smithi165 ceph-mon[25702]: Reconfiguring daemon mon.smithi165 on smithi165 2024-09-17T11:51:08.897 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:08 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T11:51:08.897 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:08 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T11:51:08.897 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:08 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:51:10.170 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:09 smithi045 ceph-mon[23737]: from='client.14250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:51:10.171 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:09 smithi045 ceph-mon[23737]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:10.171 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:09 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:10.171 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:09 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T11:51:10.171 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:09 smithi045 ceph-mon[23737]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T11:51:10.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:09 smithi165 ceph-mon[25702]: from='client.14250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:51:10.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:09 smithi165 ceph-mon[25702]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:10.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:09 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:10.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:09 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T11:51:10.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:09 smithi165 ceph-mon[25702]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T11:51:11.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:10 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/3304853145' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c063cd08-adc2-4ad1-a307-1fb7b3d2b5f3"}]: dispatch 2024-09-17T11:51:11.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:10 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/3304853145' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c063cd08-adc2-4ad1-a307-1fb7b3d2b5f3"}]': finished 2024-09-17T11:51:11.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:10 smithi165 ceph-mon[25702]: osdmap e11: 2 total, 1 up, 2 in 2024-09-17T11:51:11.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:10 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T11:51:11.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:10 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/3304853145' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c063cd08-adc2-4ad1-a307-1fb7b3d2b5f3"}]: dispatch 2024-09-17T11:51:11.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:10 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/3304853145' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c063cd08-adc2-4ad1-a307-1fb7b3d2b5f3"}]': finished 2024-09-17T11:51:11.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:10 smithi045 ceph-mon[23737]: osdmap e11: 2 total, 1 up, 2 in 2024-09-17T11:51:11.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:10 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T11:51:12.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:11 smithi165 ceph-mon[25702]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:12.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:11 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/4001890760' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T11:51:12.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:11 smithi045 ceph-mon[23737]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:12.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:11 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/4001890760' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T11:51:14.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:13 smithi165 ceph-mon[25702]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:14.291 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:13 smithi045 ceph-mon[23737]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:16.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:15 smithi165 ceph-mon[25702]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:16.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:15 smithi045 ceph-mon[23737]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:18.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:17 smithi045 ceph-mon[23737]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:18.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:17 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T11:51:18.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:17 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:51:18.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:17 smithi045 ceph-mon[23737]: Deploying daemon osd.1 on smithi045 2024-09-17T11:51:18.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:17 smithi165 ceph-mon[25702]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:18.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:17 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T11:51:18.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:17 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:51:18.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:17 smithi165 ceph-mon[25702]: Deploying daemon osd.1 on smithi045 2024-09-17T11:51:20.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:19 smithi165 ceph-mon[25702]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:20.274 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:19 smithi045 ceph-mon[23737]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:22.237 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:21 smithi045 ceph-mon[23737]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:22.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:21 smithi165 ceph-mon[25702]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:23.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:23 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:23.548 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:23 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:24.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:24 smithi165 ceph-mon[25702]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:24.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:24 smithi045 ceph-mon[23737]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:25.056 INFO:teuthology.orchestra.run.smithi045.stdout:Created osd(s) 1 on host 'smithi045' 2024-09-17T11:51:25.703 DEBUG:teuthology.orchestra.run.smithi045:osd.1> sudo journalctl -f -n 0 -u ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.1.service 2024-09-17T11:51:25.706 INFO:tasks.cephadm:Deploying osd.2 on smithi045 with /dev/vg_nvme/lv_2... 2024-09-17T11:51:25.706 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-17T11:51:26.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:26 smithi045 ceph-mon[23737]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:26.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:26 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:26.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:26 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.45:9093"}]: dispatch 2024-09-17T11:51:26.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:26 smithi045 ceph-mon[23737]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.45:9093"}]: dispatch 2024-09-17T11:51:26.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:26 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:26.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:26 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T11:51:26.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:26 smithi045 ceph-mon[23737]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T11:51:26.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:26 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.45:3000"}]: dispatch 2024-09-17T11:51:26.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:26 smithi045 ceph-mon[23737]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.45:3000"}]: dispatch 2024-09-17T11:51:26.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:26 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:26.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:26 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T11:51:26.312 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:26 smithi045 ceph-mon[23737]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T11:51:26.312 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:26 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.45:9095"}]: dispatch 2024-09-17T11:51:26.312 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:26 smithi045 ceph-mon[23737]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.45:9095"}]: dispatch 2024-09-17T11:51:26.312 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:26 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:26.312 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:26 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:51:26.312 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:26 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:51:26.312 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:26 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:51:26.312 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:26 smithi045 ceph-mon[23737]: from='osd.1 [v2:172.21.15.45:6810/2799716519,v1:172.21.15.45:6811/2799716519]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T11:51:26.333 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:26 smithi165 ceph-mon[25702]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:26.333 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:26 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:26.333 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:26 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.45:9093"}]: dispatch 2024-09-17T11:51:26.333 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:26 smithi165 ceph-mon[25702]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.45:9093"}]: dispatch 2024-09-17T11:51:26.333 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:26 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:26.333 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:26 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T11:51:26.333 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:26 smithi165 ceph-mon[25702]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T11:51:26.333 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:26 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.45:3000"}]: dispatch 2024-09-17T11:51:26.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:26 smithi165 ceph-mon[25702]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.45:3000"}]: dispatch 2024-09-17T11:51:26.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:26 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:26.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:26 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T11:51:26.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:26 smithi165 ceph-mon[25702]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T11:51:26.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:26 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.45:9095"}]: dispatch 2024-09-17T11:51:26.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:26 smithi165 ceph-mon[25702]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.45:9095"}]: dispatch 2024-09-17T11:51:26.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:26 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:26.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:26 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:51:26.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:26 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:51:26.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:26 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:51:26.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:26 smithi165 ceph-mon[25702]: from='osd.1 [v2:172.21.15.45:6810/2799716519,v1:172.21.15.45:6811/2799716519]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T11:51:27.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:27 smithi165 ceph-mon[25702]: from='osd.1 [v2:172.21.15.45:6810/2799716519,v1:172.21.15.45:6811/2799716519]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T11:51:27.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:27 smithi165 ceph-mon[25702]: osdmap e12: 2 total, 1 up, 2 in 2024-09-17T11:51:27.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:27 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T11:51:27.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:27 smithi165 ceph-mon[25702]: from='osd.1 [v2:172.21.15.45:6810/2799716519,v1:172.21.15.45:6811/2799716519]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-17T11:51:27.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:27 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:27.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:27 smithi045 ceph-mon[23737]: from='osd.1 [v2:172.21.15.45:6810/2799716519,v1:172.21.15.45:6811/2799716519]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T11:51:27.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:27 smithi045 ceph-mon[23737]: osdmap e12: 2 total, 1 up, 2 in 2024-09-17T11:51:27.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:27 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T11:51:27.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:27 smithi045 ceph-mon[23737]: from='osd.1 [v2:172.21.15.45:6810/2799716519,v1:172.21.15.45:6811/2799716519]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-17T11:51:27.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:27 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:27.590 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 11:51:27 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1[37003]: 2024-09-17T11:51:27.226+0000 7f5f038e4700 -1 osd.1 0 waiting for initial osdmap 2024-09-17T11:51:27.590 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 11:51:27 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1[37003]: 2024-09-17T11:51:27.231+0000 7f5f0027f700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T11:51:28.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:28 smithi165 ceph-mon[25702]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:28.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:28 smithi165 ceph-mon[25702]: from='osd.1 [v2:172.21.15.45:6810/2799716519,v1:172.21.15.45:6811/2799716519]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2024-09-17T11:51:28.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:28 smithi165 ceph-mon[25702]: osdmap e13: 2 total, 1 up, 2 in 2024-09-17T11:51:28.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:28 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T11:51:28.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:28 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T11:51:28.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:28 smithi045 ceph-mon[23737]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:28.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:28 smithi045 ceph-mon[23737]: from='osd.1 [v2:172.21.15.45:6810/2799716519,v1:172.21.15.45:6811/2799716519]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2024-09-17T11:51:28.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:28 smithi045 ceph-mon[23737]: osdmap e13: 2 total, 1 up, 2 in 2024-09-17T11:51:28.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:28 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T11:51:28.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:28 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T11:51:28.999 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:51:29.021 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi045:vg_nvme/lv_2 2024-09-17T11:51:29.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:29 smithi045 ceph-mon[23737]: purged_snaps scrub starts 2024-09-17T11:51:29.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:29 smithi045 ceph-mon[23737]: purged_snaps scrub ok 2024-09-17T11:51:29.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:29 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T11:51:29.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:29 smithi045 ceph-mon[23737]: osd.1 [v2:172.21.15.45:6810/2799716519,v1:172.21.15.45:6811/2799716519] boot 2024-09-17T11:51:29.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:29 smithi045 ceph-mon[23737]: osdmap e14: 2 total, 2 up, 2 in 2024-09-17T11:51:29.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:29 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T11:51:29.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:29 smithi165 ceph-mon[25702]: purged_snaps scrub starts 2024-09-17T11:51:29.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:29 smithi165 ceph-mon[25702]: purged_snaps scrub ok 2024-09-17T11:51:29.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:29 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T11:51:29.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:29 smithi165 ceph-mon[25702]: osd.1 [v2:172.21.15.45:6810/2799716519,v1:172.21.15.45:6811/2799716519] boot 2024-09-17T11:51:29.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:29 smithi165 ceph-mon[25702]: osdmap e14: 2 total, 2 up, 2 in 2024-09-17T11:51:29.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:29 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T11:51:30.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:30 smithi165 ceph-mon[25702]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:30.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:30 smithi165 ceph-mon[25702]: osdmap e15: 2 total, 2 up, 2 in 2024-09-17T11:51:30.543 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:30 smithi045 ceph-mon[23737]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T11:51:30.543 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:30 smithi045 ceph-mon[23737]: osdmap e15: 2 total, 2 up, 2 in 2024-09-17T11:51:32.504 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:32 smithi045 ceph-mon[23737]: pgmap v53: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:32.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:32 smithi165 ceph-mon[25702]: pgmap v53: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:33.375 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:33 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T11:51:33.376 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:33 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T11:51:33.376 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:33 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:51:33.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:33 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T11:51:33.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:33 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T11:51:33.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:33 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:51:34.395 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:34 smithi045 ceph-mon[23737]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:51:34.395 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:34 smithi045 ceph-mon[23737]: pgmap v54: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:34.395 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:34 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:34.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:34 smithi165 ceph-mon[25702]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:51:34.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:34 smithi165 ceph-mon[25702]: pgmap v54: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:34.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:34 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:35.508 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:35 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/1096109297' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea51869a-b1ac-483b-8f4e-8d14dc4093ad"}]: dispatch 2024-09-17T11:51:35.508 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:35 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/1096109297' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ea51869a-b1ac-483b-8f4e-8d14dc4093ad"}]': finished 2024-09-17T11:51:35.508 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:35 smithi045 ceph-mon[23737]: osdmap e16: 3 total, 2 up, 3 in 2024-09-17T11:51:35.508 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:35 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T11:51:35.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:35 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/1096109297' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea51869a-b1ac-483b-8f4e-8d14dc4093ad"}]: dispatch 2024-09-17T11:51:35.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:35 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/1096109297' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ea51869a-b1ac-483b-8f4e-8d14dc4093ad"}]': finished 2024-09-17T11:51:35.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:35 smithi165 ceph-mon[25702]: osdmap e16: 3 total, 2 up, 3 in 2024-09-17T11:51:35.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:35 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T11:51:36.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:36 smithi165 ceph-mon[25702]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:36.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:36 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/1718952714' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T11:51:36.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:36 smithi045 ceph-mon[23737]: pgmap v55: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:36.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:36 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/1718952714' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T11:51:38.511 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:38 smithi045 ceph-mon[23737]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:38.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:38 smithi165 ceph-mon[25702]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:40.397 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:40 smithi045 ceph-mon[23737]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:40.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:40 smithi165 ceph-mon[25702]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:41.973 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:41 smithi045 ceph-mon[23737]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:41.973 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:41 smithi045 ceph-mon[23737]: Detected new or changed devices on smithi045 2024-09-17T11:51:41.973 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:41 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:41.973 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:41 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:51:41.973 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:41 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:51:41.973 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:41 smithi045 ceph-mon[23737]: Adjusting osd_memory_target on smithi045 to 6434M 2024-09-17T11:51:41.973 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:41 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:41.973 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:41 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:41.973 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:41 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:51:41.974 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:41 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:51:41.974 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:41 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:51:42.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:41 smithi165 ceph-mon[25702]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:42.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:41 smithi165 ceph-mon[25702]: Detected new or changed devices on smithi045 2024-09-17T11:51:42.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:41 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:42.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:41 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:51:42.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:41 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:51:42.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:41 smithi165 ceph-mon[25702]: Adjusting osd_memory_target on smithi045 to 6434M 2024-09-17T11:51:42.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:41 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:42.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:41 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:42.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:41 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:51:42.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:41 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:51:42.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:41 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:51:43.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:43.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:43 smithi045 ceph-mon[23737]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:43.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T11:51:43.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:51:43.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:43 smithi045 ceph-mon[23737]: Deploying daemon osd.2 on smithi045 2024-09-17T11:51:43.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:43.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:43 smithi165 ceph-mon[25702]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:43.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T11:51:43.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:51:43.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:43 smithi165 ceph-mon[25702]: Deploying daemon osd.2 on smithi045 2024-09-17T11:51:45.918 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:45 smithi045 ceph-mon[23737]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:46.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:45 smithi165 ceph-mon[25702]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:47.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:47 smithi045 ceph-mon[23737]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:48.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:47 smithi165 ceph-mon[25702]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:50.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:50 smithi165 ceph-mon[25702]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:50.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:50 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:50.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:50 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:50.532 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:50 smithi045 ceph-mon[23737]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:50.532 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:50 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:50.532 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:50 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:51.447 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:51:51.447 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:51:51.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:51 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:51:51.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:51 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:51:52.304 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:52 smithi045 ceph-mon[23737]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:52.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:52 smithi165 ceph-mon[25702]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:53.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:53 smithi045 ceph-mon[23737]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:53.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:53 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:53.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:53 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:53.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:53 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:51:53.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:53 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:51:53.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:53 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:51:53.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:53 smithi045 ceph-mon[23737]: from='osd.2 [v2:172.21.15.45:6818/2178441384,v1:172.21.15.45:6819/2178441384]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T11:51:54.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:53 smithi165 ceph-mon[25702]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:54.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:53 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:54.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:53 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:54.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:53 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:51:54.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:53 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:51:54.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:53 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:51:54.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:53 smithi165 ceph-mon[25702]: from='osd.2 [v2:172.21.15.45:6818/2178441384,v1:172.21.15.45:6819/2178441384]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T11:51:55.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:54 smithi165 ceph-mon[25702]: from='osd.2 [v2:172.21.15.45:6818/2178441384,v1:172.21.15.45:6819/2178441384]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T11:51:55.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:54 smithi165 ceph-mon[25702]: osdmap e17: 3 total, 2 up, 3 in 2024-09-17T11:51:55.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:54 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T11:51:55.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:54 smithi165 ceph-mon[25702]: from='osd.2 [v2:172.21.15.45:6818/2178441384,v1:172.21.15.45:6819/2178441384]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-17T11:51:55.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:54 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:55.019 INFO:teuthology.orchestra.run.smithi045.stdout:Created osd(s) 2 on host 'smithi045' 2024-09-17T11:51:55.019 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:54 smithi045 ceph-mon[23737]: from='osd.2 [v2:172.21.15.45:6818/2178441384,v1:172.21.15.45:6819/2178441384]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T11:51:55.020 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:54 smithi045 ceph-mon[23737]: osdmap e17: 3 total, 2 up, 3 in 2024-09-17T11:51:55.020 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:54 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T11:51:55.020 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:54 smithi045 ceph-mon[23737]: from='osd.2 [v2:172.21.15.45:6818/2178441384,v1:172.21.15.45:6819/2178441384]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-17T11:51:55.020 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:54 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:55.969 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:55 smithi045 ceph-mon[23737]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:55.969 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:55 smithi045 ceph-mon[23737]: from='osd.2 [v2:172.21.15.45:6818/2178441384,v1:172.21.15.45:6819/2178441384]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2024-09-17T11:51:55.970 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:55 smithi045 ceph-mon[23737]: osdmap e18: 3 total, 2 up, 3 in 2024-09-17T11:51:55.970 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:55 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T11:51:55.970 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:55 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T11:51:55.970 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:55 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:55.998 DEBUG:teuthology.orchestra.run.smithi045:osd.2> sudo journalctl -f -n 0 -u ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.2.service 2024-09-17T11:51:56.001 INFO:tasks.cephadm:Deploying osd.3 on smithi045 with /dev/vg_nvme/lv_1... 2024-09-17T11:51:56.001 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-17T11:51:56.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:55 smithi165 ceph-mon[25702]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:56.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:55 smithi165 ceph-mon[25702]: from='osd.2 [v2:172.21.15.45:6818/2178441384,v1:172.21.15.45:6819/2178441384]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2024-09-17T11:51:56.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:55 smithi165 ceph-mon[25702]: osdmap e18: 3 total, 2 up, 3 in 2024-09-17T11:51:56.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:55 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T11:51:56.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:55 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T11:51:56.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:55 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:51:57.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:57 smithi045 ceph-mon[23737]: purged_snaps scrub starts 2024-09-17T11:51:57.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:57 smithi045 ceph-mon[23737]: purged_snaps scrub ok 2024-09-17T11:51:57.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:57 smithi045 ceph-mon[23737]: osd.2 [v2:172.21.15.45:6818/2178441384,v1:172.21.15.45:6819/2178441384] boot 2024-09-17T11:51:57.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:57 smithi045 ceph-mon[23737]: osdmap e19: 3 total, 3 up, 3 in 2024-09-17T11:51:57.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:57 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T11:51:57.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:57 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-17T11:51:57.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:57 smithi165 ceph-mon[25702]: purged_snaps scrub starts 2024-09-17T11:51:57.561 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:57 smithi165 ceph-mon[25702]: purged_snaps scrub ok 2024-09-17T11:51:57.561 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:57 smithi165 ceph-mon[25702]: osd.2 [v2:172.21.15.45:6818/2178441384,v1:172.21.15.45:6819/2178441384] boot 2024-09-17T11:51:57.561 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:57 smithi165 ceph-mon[25702]: osdmap e19: 3 total, 3 up, 3 in 2024-09-17T11:51:57.561 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:57 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T11:51:57.561 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:57 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-17T11:51:58.179 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:58 smithi045 ceph-mon[23737]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:58.179 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:58 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-17T11:51:58.179 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:58 smithi045 ceph-mon[23737]: osdmap e20: 3 total, 3 up, 3 in 2024-09-17T11:51:58.179 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:58 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-17T11:51:58.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:58 smithi165 ceph-mon[25702]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T11:51:58.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:58 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-17T11:51:58.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:58 smithi165 ceph-mon[25702]: osdmap e20: 3 total, 3 up, 3 in 2024-09-17T11:51:58.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:58 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-17T11:51:59.089 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 11:51:58 smithi045 sudo[42050]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T11:51:59.089 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 11:51:58 smithi045 sudo[42050]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T11:51:59.090 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 11:51:58 smithi045 sudo[42050]: pam_unix(sudo:session): session closed for user root 2024-09-17T11:51:59.473 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:59 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-17T11:51:59.473 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:51:59 smithi045 ceph-mon[23737]: osdmap e21: 3 total, 3 up, 3 in 2024-09-17T11:51:59.473 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 11:51:59 smithi045 sudo[42054]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T11:51:59.473 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 11:51:59 smithi045 sudo[42054]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T11:51:59.473 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 11:51:59 smithi045 sudo[42054]: pam_unix(sudo:session): session closed for user root 2024-09-17T11:51:59.478 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:51:59.496 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi045:vg_nvme/lv_1 2024-09-17T11:51:59.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:59 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-17T11:51:59.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:51:59 smithi165 ceph-mon[25702]: osdmap e21: 3 total, 3 up, 3 in 2024-09-17T11:52:00.005 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 11:51:59 smithi045 sudo[42120]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T11:52:00.006 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 11:51:59 smithi045 sudo[42120]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T11:52:00.006 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 11:51:59 smithi045 sudo[42120]: pam_unix(sudo:session): session closed for user root 2024-09-17T11:52:00.339 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 11:52:00 smithi045 sudo[42155]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T11:52:00.339 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 11:52:00 smithi045 sudo[42155]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T11:52:00.339 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 11:52:00 smithi045 sudo[42155]: pam_unix(sudo:session): session closed for user root 2024-09-17T11:52:00.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:00 smithi045 ceph-mon[23737]: pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T11:52:00.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:00 smithi045 ceph-mon[23737]: osdmap e22: 3 total, 3 up, 3 in 2024-09-17T11:52:00.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:00 smithi045 ceph-mon[23737]: mgrmap e19: smithi045.jtwyrp(active, since 2m), standbys: smithi165.ykdszc 2024-09-17T11:52:00.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:00 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:00.498 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:00 smithi165 ceph-mon[25702]: pgmap v73: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T11:52:00.499 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:00 smithi165 ceph-mon[25702]: osdmap e22: 3 total, 3 up, 3 in 2024-09-17T11:52:00.499 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:00 smithi165 ceph-mon[25702]: mgrmap e19: smithi045.jtwyrp(active, since 2m), standbys: smithi165.ykdszc 2024-09-17T11:52:00.499 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:00 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:00.881 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 11:52:00 smithi045 sudo[42174]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T11:52:00.881 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 11:52:00 smithi045 sudo[42174]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T11:52:00.881 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 11:52:00 smithi045 sudo[42174]: pam_unix(sudo:session): session closed for user root 2024-09-17T11:52:01.153 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 11:52:01 smithi045 sudo[42195]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T11:52:01.154 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 11:52:01 smithi045 sudo[42195]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T11:52:01.154 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 11:52:01 smithi045 sudo[42195]: pam_unix(sudo:session): session closed for user root 2024-09-17T11:52:01.820 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:01 smithi045 sudo[42223]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T11:52:01.820 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:01 smithi045 sudo[42223]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T11:52:01.820 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:01 smithi045 sudo[42223]: pam_unix(sudo:session): session closed for user root 2024-09-17T11:52:02.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:01 smithi045 sudo[42240]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T11:52:02.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:01 smithi045 sudo[42240]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T11:52:02.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:01 smithi045 sudo[42240]: pam_unix(sudo:session): session closed for user root 2024-09-17T11:52:02.484 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:02 smithi165 sudo[28025]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T11:52:02.484 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:02 smithi165 sudo[28025]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T11:52:02.484 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:02 smithi165 sudo[28025]: pam_unix(sudo:session): session closed for user root 2024-09-17T11:52:02.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:02 smithi165 sudo[28028]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T11:52:02.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:02 smithi165 sudo[28028]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T11:52:02.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:02 smithi165 sudo[28028]: pam_unix(sudo:session): session closed for user root 2024-09-17T11:52:02.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:02 smithi165 ceph-mon[25702]: pgmap v75: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-17T11:52:02.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:02 smithi165 ceph-mon[25702]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T11:52:02.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:02 smithi165 ceph-mon[25702]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T11:52:02.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:02 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi045"}]: dispatch 2024-09-17T11:52:02.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:02 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi165"}]: dispatch 2024-09-17T11:52:02.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:02 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi045"}]: dispatch 2024-09-17T11:52:02.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:02 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi165"}]: dispatch 2024-09-17T11:52:02.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:02 smithi045 ceph-mon[23737]: pgmap v75: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-17T11:52:02.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:02 smithi045 ceph-mon[23737]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T11:52:02.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:02 smithi045 ceph-mon[23737]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T11:52:02.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:02 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi045"}]: dispatch 2024-09-17T11:52:02.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:02 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi165"}]: dispatch 2024-09-17T11:52:02.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:02 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi045"}]: dispatch 2024-09-17T11:52:02.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:02 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi165"}]: dispatch 2024-09-17T11:52:03.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:03 smithi165 ceph-mon[25702]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T11:52:03.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:03 smithi165 ceph-mon[25702]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:52:03.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:03 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T11:52:03.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:03 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T11:52:03.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:03 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:52:03.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:03 smithi165 ceph-mon[25702]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T11:52:03.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:03 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:03.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:03 smithi165 ceph-mon[25702]: pgmap v76: 1 pgs: 1 active+clean; 449 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-17T11:52:03.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:03 smithi045 ceph-mon[23737]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T11:52:03.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:03 smithi045 ceph-mon[23737]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi045:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:52:03.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:03 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T11:52:03.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:03 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T11:52:03.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:03 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:52:03.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:03 smithi045 ceph-mon[23737]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T11:52:03.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:03 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:03.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:03 smithi045 ceph-mon[23737]: pgmap v76: 1 pgs: 1 active+clean; 449 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-17T11:52:05.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:05 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:05.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:05 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/1269594650' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fbc72a30-0ec4-4738-86fb-b0ec8993e695"}]: dispatch 2024-09-17T11:52:05.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:05 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/1269594650' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fbc72a30-0ec4-4738-86fb-b0ec8993e695"}]': finished 2024-09-17T11:52:05.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:05 smithi165 ceph-mon[25702]: osdmap e23: 4 total, 3 up, 4 in 2024-09-17T11:52:05.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:05 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T11:52:05.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:05 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:05.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:05 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/1269594650' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fbc72a30-0ec4-4738-86fb-b0ec8993e695"}]: dispatch 2024-09-17T11:52:05.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:05 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/1269594650' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fbc72a30-0ec4-4738-86fb-b0ec8993e695"}]': finished 2024-09-17T11:52:05.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:05 smithi045 ceph-mon[23737]: osdmap e23: 4 total, 3 up, 4 in 2024-09-17T11:52:05.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:05 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T11:52:06.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:06 smithi165 ceph-mon[25702]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 6.7 KiB/s rd, 326 KiB/s wr, 22 op/s 2024-09-17T11:52:06.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:06 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/202135115' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T11:52:06.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:06 smithi045 ceph-mon[23737]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 6.7 KiB/s rd, 326 KiB/s wr, 22 op/s 2024-09-17T11:52:06.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:06 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/202135115' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T11:52:08.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:08 smithi165 ceph-mon[25702]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 218 KiB/s wr, 14 op/s 2024-09-17T11:52:08.563 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:08 smithi045 ceph-mon[23737]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 218 KiB/s wr, 14 op/s 2024-09-17T11:52:10.423 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:10 smithi045 ceph-mon[23737]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.4 KiB/s rd, 163 KiB/s wr, 11 op/s 2024-09-17T11:52:10.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:10 smithi165 ceph-mon[25702]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.4 KiB/s rd, 163 KiB/s wr, 11 op/s 2024-09-17T11:52:11.658 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:11 smithi045 ceph-mon[23737]: Detected new or changed devices on smithi045 2024-09-17T11:52:11.658 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:11 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:11.658 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:11 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:52:11.658 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:11 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:52:11.658 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:11 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:52:11.658 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:11 smithi045 ceph-mon[23737]: Adjusting osd_memory_target on smithi045 to 4289M 2024-09-17T11:52:11.658 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:11 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:11.658 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:11 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:11.658 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:11 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:52:11.658 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:11 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:52:11.658 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:11 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:52:11.659 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:11 smithi045 ceph-mon[23737]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-09-17T11:52:11.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:11 smithi165 ceph-mon[25702]: Detected new or changed devices on smithi045 2024-09-17T11:52:11.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:11 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:11.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:11 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:52:11.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:11 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:52:11.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:11 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:52:11.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:11 smithi165 ceph-mon[25702]: Adjusting osd_memory_target on smithi045 to 4289M 2024-09-17T11:52:11.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:11 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:11.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:11 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:11.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:11 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:52:11.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:11 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:52:11.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:11 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:52:11.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:11 smithi165 ceph-mon[25702]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-09-17T11:52:12.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:12 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T11:52:12.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:12 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:52:12.666 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:12 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T11:52:12.667 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:12 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:52:13.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:13 smithi045 ceph-mon[23737]: Deploying daemon osd.3 on smithi045 2024-09-17T11:52:13.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:13 smithi045 ceph-mon[23737]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-09-17T11:52:14.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:13 smithi165 ceph-mon[25702]: Deploying daemon osd.3 on smithi045 2024-09-17T11:52:14.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:13 smithi165 ceph-mon[25702]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-09-17T11:52:16.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:15 smithi165 ceph-mon[25702]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T11:52:16.026 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:15 smithi045 ceph-mon[23737]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T11:52:18.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:17 smithi165 ceph-mon[25702]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T11:52:18.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:17 smithi045 ceph-mon[23737]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T11:52:19.026 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:18 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:19.027 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:18 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:19.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:18 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:19.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:18 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:20.142 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:19 smithi045 ceph-mon[23737]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T11:52:20.142 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:19 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:20.142 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:19 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:52:20.142 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:19 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:52:20.142 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:19 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:52:20.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:19 smithi165 ceph-mon[25702]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T11:52:20.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:19 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:20.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:19 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:52:20.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:19 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:52:20.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:19 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:52:22.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:21 smithi045 ceph-mon[23737]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T11:52:22.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:21 smithi165 ceph-mon[25702]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T11:52:22.855 INFO:teuthology.orchestra.run.smithi045.stdout:Created osd(s) 3 on host 'smithi045' 2024-09-17T11:52:23.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:22 smithi045 ceph-mon[23737]: from='osd.3 [v2:172.21.15.45:6826/2915608972,v1:172.21.15.45:6827/2915608972]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T11:52:23.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:22 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:23.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:22 smithi165 ceph-mon[25702]: from='osd.3 [v2:172.21.15.45:6826/2915608972,v1:172.21.15.45:6827/2915608972]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T11:52:23.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:22 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:24.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:23 smithi045 ceph-mon[23737]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T11:52:24.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:23 smithi045 ceph-mon[23737]: from='osd.3 [v2:172.21.15.45:6826/2915608972,v1:172.21.15.45:6827/2915608972]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T11:52:24.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:23 smithi045 ceph-mon[23737]: osdmap e24: 4 total, 3 up, 4 in 2024-09-17T11:52:24.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:23 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T11:52:24.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:23 smithi045 ceph-mon[23737]: from='osd.3 [v2:172.21.15.45:6826/2915608972,v1:172.21.15.45:6827/2915608972]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-17T11:52:24.175 DEBUG:teuthology.orchestra.run.smithi045:osd.3> sudo journalctl -f -n 0 -u ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.3.service 2024-09-17T11:52:24.178 INFO:tasks.cephadm:Deploying osd.4 on smithi165 with /dev/vg_nvme/lv_4... 2024-09-17T11:52:24.178 DEBUG:teuthology.orchestra.run.smithi165:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-17T11:52:24.207 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:23 smithi165 ceph-mon[25702]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T11:52:24.208 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:23 smithi165 ceph-mon[25702]: from='osd.3 [v2:172.21.15.45:6826/2915608972,v1:172.21.15.45:6827/2915608972]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T11:52:24.208 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:23 smithi165 ceph-mon[25702]: osdmap e24: 4 total, 3 up, 4 in 2024-09-17T11:52:24.208 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:23 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T11:52:24.208 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:23 smithi165 ceph-mon[25702]: from='osd.3 [v2:172.21.15.45:6826/2915608972,v1:172.21.15.45:6827/2915608972]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]: dispatch 2024-09-17T11:52:25.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:24 smithi165 ceph-mon[25702]: from='osd.3 [v2:172.21.15.45:6826/2915608972,v1:172.21.15.45:6827/2915608972]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2024-09-17T11:52:25.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:24 smithi165 ceph-mon[25702]: osdmap e25: 4 total, 3 up, 4 in 2024-09-17T11:52:25.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:24 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T11:52:25.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:24 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T11:52:25.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:24 smithi045 ceph-mon[23737]: from='osd.3 [v2:172.21.15.45:6826/2915608972,v1:172.21.15.45:6827/2915608972]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi045", "root=default"]}]': finished 2024-09-17T11:52:25.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:24 smithi045 ceph-mon[23737]: osdmap e25: 4 total, 3 up, 4 in 2024-09-17T11:52:25.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:24 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T11:52:25.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:24 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T11:52:26.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:25 smithi165 ceph-mon[25702]: purged_snaps scrub starts 2024-09-17T11:52:26.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:25 smithi165 ceph-mon[25702]: purged_snaps scrub ok 2024-09-17T11:52:26.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:25 smithi165 ceph-mon[25702]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T11:52:26.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:25 smithi165 ceph-mon[25702]: osd.3 [v2:172.21.15.45:6826/2915608972,v1:172.21.15.45:6827/2915608972] boot 2024-09-17T11:52:26.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:25 smithi165 ceph-mon[25702]: osdmap e26: 4 total, 4 up, 4 in 2024-09-17T11:52:26.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:25 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T11:52:26.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:25 smithi045 ceph-mon[23737]: purged_snaps scrub starts 2024-09-17T11:52:26.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:25 smithi045 ceph-mon[23737]: purged_snaps scrub ok 2024-09-17T11:52:26.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:25 smithi045 ceph-mon[23737]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T11:52:26.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:25 smithi045 ceph-mon[23737]: osd.3 [v2:172.21.15.45:6826/2915608972,v1:172.21.15.45:6827/2915608972] boot 2024-09-17T11:52:26.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:25 smithi045 ceph-mon[23737]: osdmap e26: 4 total, 4 up, 4 in 2024-09-17T11:52:26.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:25 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T11:52:26.606 INFO:teuthology.orchestra.run.smithi165.stdout: 2024-09-17T11:52:26.626 DEBUG:teuthology.orchestra.run.smithi165:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi165:vg_nvme/lv_4 2024-09-17T11:52:27.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:26 smithi045 ceph-mon[23737]: osdmap e27: 4 total, 4 up, 4 in 2024-09-17T11:52:27.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:26 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:27.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:26 smithi165 ceph-mon[25702]: osdmap e27: 4 total, 4 up, 4 in 2024-09-17T11:52:27.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:26 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:28.150 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:27 smithi045 ceph-mon[23737]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T11:52:28.166 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:27 smithi165 ceph-mon[25702]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T11:52:28.973 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:28 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T11:52:28.974 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:28 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T11:52:28.974 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:28 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:52:29.186 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:28 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T11:52:29.186 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:28 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T11:52:29.186 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:28 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:52:30.208 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:29 smithi165 ceph-mon[25702]: from='client.24153 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi165:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:52:30.209 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:29 smithi165 ceph-mon[25702]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T11:52:30.209 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:29 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:30.209 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:29 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:52:30.209 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:29 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:52:30.209 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:29 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:52:30.209 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:29 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:52:30.209 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:29 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:30.209 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:29 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:30.209 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:29 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:52:30.209 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:29 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:52:30.209 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:29 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:52:30.282 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:29 smithi045 ceph-mon[23737]: from='client.24153 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi165:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:52:30.282 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:29 smithi045 ceph-mon[23737]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T11:52:30.282 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:29 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:30.282 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:29 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:52:30.283 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:29 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:52:30.283 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:29 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:52:30.283 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:29 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:52:30.283 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:29 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:30.283 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:29 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:30.283 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:29 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:52:30.283 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:29 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:52:30.283 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:29 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:52:31.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:30 smithi165 ceph-mon[25702]: Adjusting osd_memory_target on smithi045 to 3217M 2024-09-17T11:52:31.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:30 smithi165 ceph-mon[25702]: from='client.? 172.21.15.165:0/4002227437' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e538740a-c502-4d61-83b2-1847a9083ff5"}]: dispatch 2024-09-17T11:52:31.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:30 smithi165 ceph-mon[25702]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e538740a-c502-4d61-83b2-1847a9083ff5"}]: dispatch 2024-09-17T11:52:31.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:30 smithi165 ceph-mon[25702]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e538740a-c502-4d61-83b2-1847a9083ff5"}]': finished 2024-09-17T11:52:31.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:30 smithi165 ceph-mon[25702]: osdmap e28: 5 total, 4 up, 5 in 2024-09-17T11:52:31.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:30 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T11:52:31.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:30 smithi045 ceph-mon[23737]: Adjusting osd_memory_target on smithi045 to 3217M 2024-09-17T11:52:31.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:30 smithi045 ceph-mon[23737]: from='client.? 172.21.15.165:0/4002227437' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e538740a-c502-4d61-83b2-1847a9083ff5"}]: dispatch 2024-09-17T11:52:31.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:30 smithi045 ceph-mon[23737]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e538740a-c502-4d61-83b2-1847a9083ff5"}]: dispatch 2024-09-17T11:52:31.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:30 smithi045 ceph-mon[23737]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e538740a-c502-4d61-83b2-1847a9083ff5"}]': finished 2024-09-17T11:52:31.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:30 smithi045 ceph-mon[23737]: osdmap e28: 5 total, 4 up, 5 in 2024-09-17T11:52:31.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:30 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T11:52:32.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:31 smithi165 ceph-mon[25702]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T11:52:32.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:31 smithi165 ceph-mon[25702]: from='client.? 172.21.15.165:0/3380476102' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T11:52:32.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:31 smithi045 ceph-mon[23737]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T11:52:32.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:31 smithi045 ceph-mon[23737]: from='client.? 172.21.15.165:0/3380476102' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T11:52:34.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:33 smithi165 ceph-mon[25702]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T11:52:34.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:33 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:34.068 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:33 smithi045 ceph-mon[23737]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T11:52:34.068 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:33 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:35.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:35 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:35.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:35 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:36.213 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:36 smithi165 ceph-mon[25702]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T11:52:36.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:36 smithi045 ceph-mon[23737]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T11:52:37.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:37 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T11:52:37.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:37 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:52:37.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:37 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T11:52:37.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:37 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:52:38.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:38 smithi045 ceph-mon[23737]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T11:52:38.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:38 smithi045 ceph-mon[23737]: Deploying daemon osd.4 on smithi165 2024-09-17T11:52:38.378 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:38 smithi165 ceph-mon[25702]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T11:52:38.378 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:38 smithi165 ceph-mon[25702]: Deploying daemon osd.4 on smithi165 2024-09-17T11:52:40.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:40 smithi045 ceph-mon[23737]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T11:52:40.382 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:40 smithi165 ceph-mon[25702]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T11:52:42.345 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:42 smithi165 ceph-mon[25702]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T11:52:42.345 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:42 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:42.345 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:42 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:52:42.345 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:42 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:52:42.345 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:42 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:52:42.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:42 smithi045 ceph-mon[23737]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T11:52:42.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:42 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:42.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:42 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:52:42.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:42 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:52:42.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:42 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:52:44.150 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:44 smithi165 ceph-mon[25702]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T11:52:44.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:44 smithi045 ceph-mon[23737]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T11:52:45.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:45 smithi165 ceph-mon[25702]: from='osd.4 [v2:172.21.15.165:6800/2995343118,v1:172.21.15.165:6801/2995343118]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T11:52:45.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:45 smithi165 ceph-mon[25702]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T11:52:45.587 INFO:teuthology.orchestra.run.smithi165.stdout:Created osd(s) 4 on host 'smithi165' 2024-09-17T11:52:45.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:45 smithi045 ceph-mon[23737]: from='osd.4 [v2:172.21.15.165:6800/2995343118,v1:172.21.15.165:6801/2995343118]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T11:52:45.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:45 smithi045 ceph-mon[23737]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T11:52:46.200 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:46 smithi165 ceph-mon[25702]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T11:52:46.200 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:46 smithi165 ceph-mon[25702]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T11:52:46.200 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:46 smithi165 ceph-mon[25702]: osdmap e29: 5 total, 4 up, 5 in 2024-09-17T11:52:46.200 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:46 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T11:52:46.200 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:46 smithi165 ceph-mon[25702]: from='osd.4 [v2:172.21.15.165:6800/2995343118,v1:172.21.15.165:6801/2995343118]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]: dispatch 2024-09-17T11:52:46.200 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:46 smithi165 ceph-mon[25702]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]: dispatch 2024-09-17T11:52:46.201 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:46 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:46.201 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:46 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:46.221 DEBUG:teuthology.orchestra.run.smithi165:osd.4> sudo journalctl -f -n 0 -u ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.4.service 2024-09-17T11:52:46.223 INFO:tasks.cephadm:Deploying osd.5 on smithi165 with /dev/vg_nvme/lv_3... 2024-09-17T11:52:46.224 DEBUG:teuthology.orchestra.run.smithi165:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-17T11:52:46.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:46 smithi045 ceph-mon[23737]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T11:52:46.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:46 smithi045 ceph-mon[23737]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T11:52:46.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:46 smithi045 ceph-mon[23737]: osdmap e29: 5 total, 4 up, 5 in 2024-09-17T11:52:46.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:46 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T11:52:46.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:46 smithi045 ceph-mon[23737]: from='osd.4 [v2:172.21.15.165:6800/2995343118,v1:172.21.15.165:6801/2995343118]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]: dispatch 2024-09-17T11:52:46.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:46 smithi045 ceph-mon[23737]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]: dispatch 2024-09-17T11:52:46.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:46 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:46.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:46 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:47.207 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:47 smithi165 ceph-mon[25702]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]': finished 2024-09-17T11:52:47.207 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:47 smithi165 ceph-mon[25702]: osdmap e30: 5 total, 4 up, 5 in 2024-09-17T11:52:47.207 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:47 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T11:52:47.207 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:47 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T11:52:47.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:47 smithi045 ceph-mon[23737]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]': finished 2024-09-17T11:52:47.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:47 smithi045 ceph-mon[23737]: osdmap e30: 5 total, 4 up, 5 in 2024-09-17T11:52:47.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:47 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T11:52:47.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:47 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T11:52:48.427 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:48 smithi165 ceph-mon[25702]: purged_snaps scrub starts 2024-09-17T11:52:48.427 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:48 smithi165 ceph-mon[25702]: purged_snaps scrub ok 2024-09-17T11:52:48.427 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:48 smithi165 ceph-mon[25702]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T11:52:48.427 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:48 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T11:52:48.428 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:48 smithi165 ceph-mon[25702]: osd.4 [v2:172.21.15.165:6800/2995343118,v1:172.21.15.165:6801/2995343118] boot 2024-09-17T11:52:48.428 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:48 smithi165 ceph-mon[25702]: osdmap e31: 5 total, 5 up, 5 in 2024-09-17T11:52:48.428 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:48 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T11:52:48.428 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:48 smithi165 ceph-mon[25702]: osdmap e32: 5 total, 5 up, 5 in 2024-09-17T11:52:48.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:48 smithi045 ceph-mon[23737]: purged_snaps scrub starts 2024-09-17T11:52:48.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:48 smithi045 ceph-mon[23737]: purged_snaps scrub ok 2024-09-17T11:52:48.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:48 smithi045 ceph-mon[23737]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T11:52:48.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:48 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T11:52:48.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:48 smithi045 ceph-mon[23737]: osd.4 [v2:172.21.15.165:6800/2995343118,v1:172.21.15.165:6801/2995343118] boot 2024-09-17T11:52:48.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:48 smithi045 ceph-mon[23737]: osdmap e31: 5 total, 5 up, 5 in 2024-09-17T11:52:48.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:48 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T11:52:48.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:48 smithi045 ceph-mon[23737]: osdmap e32: 5 total, 5 up, 5 in 2024-09-17T11:52:50.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:49 smithi165 ceph-mon[25702]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-17T11:52:50.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:49 smithi165 ceph-mon[25702]: osdmap e33: 5 total, 5 up, 5 in 2024-09-17T11:52:50.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:49 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:50.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:49 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:52:50.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:49 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:50.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:49 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:50.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:49 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:52:50.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:49 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:52:50.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:49 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:52:50.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:49 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:50.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:49 smithi045 ceph-mon[23737]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-17T11:52:50.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:49 smithi045 ceph-mon[23737]: osdmap e33: 5 total, 5 up, 5 in 2024-09-17T11:52:50.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:49 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:50.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:49 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:52:50.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:49 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:50.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:49 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:50.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:49 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:52:50.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:49 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:52:50.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:49 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:52:50.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:49 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:52:50.723 INFO:teuthology.orchestra.run.smithi165.stdout: 2024-09-17T11:52:50.742 DEBUG:teuthology.orchestra.run.smithi165:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi165:vg_nvme/lv_3 2024-09-17T11:52:51.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:50 smithi165 ceph-mon[25702]: Detected new or changed devices on smithi165 2024-09-17T11:52:51.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:50 smithi165 ceph-mon[25702]: Adjusting osd_memory_target on smithi165 to 15936M 2024-09-17T11:52:51.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:50 smithi165 ceph-mon[25702]: osdmap e34: 5 total, 5 up, 5 in 2024-09-17T11:52:51.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:50 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:52:51.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:50 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:52:51.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:50 smithi045 ceph-mon[23737]: Detected new or changed devices on smithi165 2024-09-17T11:52:51.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:50 smithi045 ceph-mon[23737]: Adjusting osd_memory_target on smithi165 to 15936M 2024-09-17T11:52:51.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:50 smithi045 ceph-mon[23737]: osdmap e34: 5 total, 5 up, 5 in 2024-09-17T11:52:51.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:50 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:52:51.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:50 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:52:52.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:51 smithi165 ceph-mon[25702]: pgmap v112: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-17T11:52:52.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:51 smithi165 ceph-mon[25702]: osdmap e35: 5 total, 5 up, 5 in 2024-09-17T11:52:52.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:51 smithi045 ceph-mon[23737]: pgmap v112: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-17T11:52:52.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:51 smithi045 ceph-mon[23737]: osdmap e35: 5 total, 5 up, 5 in 2024-09-17T11:52:52.888 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:52 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T11:52:52.888 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:52 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T11:52:52.888 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:52 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:52:52.991 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:52 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T11:52:52.991 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:52 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T11:52:52.991 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:52 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:52:54.003 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:53 smithi045 ceph-mon[23737]: from='client.24171 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi165:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:52:54.003 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:53 smithi045 ceph-mon[23737]: pgmap v114: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-09-17T11:52:54.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:53 smithi165 ceph-mon[25702]: from='client.24171 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi165:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:52:54.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:53 smithi165 ceph-mon[25702]: pgmap v114: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-09-17T11:52:56.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:55 smithi165 ceph-mon[25702]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T11:52:56.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:55 smithi165 ceph-mon[25702]: from='client.? 172.21.15.165:0/1902142212' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "20205cf8-a248-4b9d-9e97-c43f8ded5924"}]: dispatch 2024-09-17T11:52:56.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:55 smithi165 ceph-mon[25702]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "20205cf8-a248-4b9d-9e97-c43f8ded5924"}]: dispatch 2024-09-17T11:52:56.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:55 smithi165 ceph-mon[25702]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "20205cf8-a248-4b9d-9e97-c43f8ded5924"}]': finished 2024-09-17T11:52:56.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:55 smithi165 ceph-mon[25702]: osdmap e36: 6 total, 5 up, 6 in 2024-09-17T11:52:56.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:55 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T11:52:56.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:55 smithi165 ceph-mon[25702]: from='client.? 172.21.15.165:0/2831969102' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T11:52:56.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:55 smithi045 ceph-mon[23737]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T11:52:56.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:55 smithi045 ceph-mon[23737]: from='client.? 172.21.15.165:0/1902142212' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "20205cf8-a248-4b9d-9e97-c43f8ded5924"}]: dispatch 2024-09-17T11:52:56.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:55 smithi045 ceph-mon[23737]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "20205cf8-a248-4b9d-9e97-c43f8ded5924"}]: dispatch 2024-09-17T11:52:56.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:55 smithi045 ceph-mon[23737]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "20205cf8-a248-4b9d-9e97-c43f8ded5924"}]': finished 2024-09-17T11:52:56.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:55 smithi045 ceph-mon[23737]: osdmap e36: 6 total, 5 up, 6 in 2024-09-17T11:52:56.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:55 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T11:52:56.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:55 smithi045 ceph-mon[23737]: from='client.? 172.21.15.165:0/2831969102' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T11:52:58.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:57 smithi165 ceph-mon[25702]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T11:52:58.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:57 smithi045 ceph-mon[23737]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T11:52:59.879 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:52:59 smithi165 ceph-mon[25702]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T11:53:00.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:52:59 smithi045 ceph-mon[23737]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T11:53:02.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:01 smithi165 ceph-mon[25702]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T11:53:02.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:01 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T11:53:02.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:01 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:53:02.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:01 smithi165 ceph-mon[25702]: Deploying daemon osd.5 on smithi165 2024-09-17T11:53:02.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:01 smithi045 ceph-mon[23737]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T11:53:02.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:01 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T11:53:02.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:01 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:53:02.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:01 smithi045 ceph-mon[23737]: Deploying daemon osd.5 on smithi165 2024-09-17T11:53:03.987 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:03 smithi165 ceph-mon[25702]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T11:53:04.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:03 smithi045 ceph-mon[23737]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T11:53:06.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:05 smithi165 ceph-mon[25702]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T11:53:06.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:05 smithi045 ceph-mon[23737]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T11:53:07.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:06 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:53:07.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:06 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:07.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:06 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:53:07.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:06 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:53:07.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:06 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:53:07.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:06 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:07.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:06 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:53:07.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:06 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:53:08.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:08 smithi165 ceph-mon[25702]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T11:53:08.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:08 smithi045 ceph-mon[23737]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T11:53:09.045 INFO:teuthology.orchestra.run.smithi165.stdout:Created osd(s) 5 on host 'smithi165' 2024-09-17T11:53:09.648 DEBUG:teuthology.orchestra.run.smithi165:osd.5> sudo journalctl -f -n 0 -u ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.5.service 2024-09-17T11:53:09.651 INFO:tasks.cephadm:Deploying osd.6 on smithi165 with /dev/vg_nvme/lv_2... 2024-09-17T11:53:09.651 DEBUG:teuthology.orchestra.run.smithi165:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-17T11:53:10.201 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:10 smithi165 ceph-mon[25702]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T11:53:10.202 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:10 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:10.202 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:10 smithi165 ceph-mon[25702]: from='osd.5 [v2:172.21.15.165:6808/2948491063,v1:172.21.15.165:6809/2948491063]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T11:53:10.202 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:10 smithi165 ceph-mon[25702]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T11:53:10.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:10 smithi045 ceph-mon[23737]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T11:53:10.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:10 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:10.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:10 smithi045 ceph-mon[23737]: from='osd.5 [v2:172.21.15.165:6808/2948491063,v1:172.21.15.165:6809/2948491063]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T11:53:10.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:10 smithi045 ceph-mon[23737]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T11:53:11.500 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:11 smithi165 ceph-mon[25702]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T11:53:11.500 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:11 smithi165 ceph-mon[25702]: osdmap e37: 6 total, 5 up, 6 in 2024-09-17T11:53:11.500 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:11 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T11:53:11.500 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:11 smithi165 ceph-mon[25702]: from='osd.5 [v2:172.21.15.165:6808/2948491063,v1:172.21.15.165:6809/2948491063]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]: dispatch 2024-09-17T11:53:11.501 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:11 smithi165 ceph-mon[25702]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]: dispatch 2024-09-17T11:53:11.501 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:11 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:11.501 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 11:53:11 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[32880]: 2024-09-17T11:53:11.104+0000 7f6f1c832700 -1 osd.5 0 waiting for initial osdmap 2024-09-17T11:53:11.501 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 11:53:11 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[32880]: 2024-09-17T11:53:11.109+0000 7f6f171c9700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T11:53:11.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:11 smithi045 ceph-mon[23737]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T11:53:11.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:11 smithi045 ceph-mon[23737]: osdmap e37: 6 total, 5 up, 6 in 2024-09-17T11:53:11.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:11 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T11:53:11.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:11 smithi045 ceph-mon[23737]: from='osd.5 [v2:172.21.15.165:6808/2948491063,v1:172.21.15.165:6809/2948491063]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]: dispatch 2024-09-17T11:53:11.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:11 smithi045 ceph-mon[23737]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]: dispatch 2024-09-17T11:53:11.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:11 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:12.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:12 smithi165 ceph-mon[25702]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T11:53:12.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:12 smithi165 ceph-mon[25702]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]': finished 2024-09-17T11:53:12.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:12 smithi165 ceph-mon[25702]: osdmap e38: 6 total, 5 up, 6 in 2024-09-17T11:53:12.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:12 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T11:53:12.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:12 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T11:53:12.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:12 smithi045 ceph-mon[23737]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T11:53:12.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:12 smithi045 ceph-mon[23737]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]': finished 2024-09-17T11:53:12.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:12 smithi045 ceph-mon[23737]: osdmap e38: 6 total, 5 up, 6 in 2024-09-17T11:53:12.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:12 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T11:53:12.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:12 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T11:53:12.822 INFO:teuthology.orchestra.run.smithi165.stdout: 2024-09-17T11:53:12.840 DEBUG:teuthology.orchestra.run.smithi165:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi165:vg_nvme/lv_2 2024-09-17T11:53:13.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:13 smithi165 ceph-mon[25702]: purged_snaps scrub starts 2024-09-17T11:53:13.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:13 smithi165 ceph-mon[25702]: purged_snaps scrub ok 2024-09-17T11:53:13.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:13 smithi165 ceph-mon[25702]: osd.5 [v2:172.21.15.165:6808/2948491063,v1:172.21.15.165:6809/2948491063] boot 2024-09-17T11:53:13.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:13 smithi165 ceph-mon[25702]: osdmap e39: 6 total, 6 up, 6 in 2024-09-17T11:53:13.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:13 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T11:53:13.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:13 smithi165 ceph-mon[25702]: osdmap e40: 6 total, 6 up, 6 in 2024-09-17T11:53:13.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:13 smithi045 ceph-mon[23737]: purged_snaps scrub starts 2024-09-17T11:53:13.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:13 smithi045 ceph-mon[23737]: purged_snaps scrub ok 2024-09-17T11:53:13.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:13 smithi045 ceph-mon[23737]: osd.5 [v2:172.21.15.165:6808/2948491063,v1:172.21.15.165:6809/2948491063] boot 2024-09-17T11:53:13.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:13 smithi045 ceph-mon[23737]: osdmap e39: 6 total, 6 up, 6 in 2024-09-17T11:53:13.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:13 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T11:53:13.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:13 smithi045 ceph-mon[23737]: osdmap e40: 6 total, 6 up, 6 in 2024-09-17T11:53:14.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:14 smithi165 ceph-mon[25702]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T11:53:14.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:14 smithi165 ceph-mon[25702]: osdmap e41: 6 total, 6 up, 6 in 2024-09-17T11:53:14.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:14 smithi045 ceph-mon[23737]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T11:53:14.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:14 smithi045 ceph-mon[23737]: osdmap e41: 6 total, 6 up, 6 in 2024-09-17T11:53:16.001 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:15 smithi165 ceph-mon[25702]: pgmap v131: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T11:53:16.002 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:15 smithi165 ceph-mon[25702]: osdmap e42: 6 total, 6 up, 6 in 2024-09-17T11:53:16.002 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:15 smithi165 ceph-mon[25702]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T11:53:16.002 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:15 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:16.002 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:15 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:53:16.002 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:15 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:53:16.002 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:15 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:16.002 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:15 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:16.002 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:15 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:53:16.002 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:15 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:53:16.002 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:15 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:53:16.002 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:15 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:16.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:15 smithi045 ceph-mon[23737]: pgmap v131: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T11:53:16.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:15 smithi045 ceph-mon[23737]: osdmap e42: 6 total, 6 up, 6 in 2024-09-17T11:53:16.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:15 smithi045 ceph-mon[23737]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T11:53:16.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:15 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:16.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:15 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:53:16.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:15 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:53:16.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:15 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:16.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:15 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:16.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:15 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:53:16.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:15 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:53:16.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:15 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:53:16.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:15 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:16.913 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:16 smithi165 ceph-mon[25702]: Detected new or changed devices on smithi165 2024-09-17T11:53:16.913 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:16 smithi165 ceph-mon[25702]: Adjusting osd_memory_target on smithi165 to 7968M 2024-09-17T11:53:16.913 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:16 smithi165 ceph-mon[25702]: osdmap e43: 6 total, 6 up, 6 in 2024-09-17T11:53:16.913 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:16 smithi165 ceph-mon[25702]: from='client.24189 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi165:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:53:16.913 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:16 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T11:53:16.913 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:16 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T11:53:16.914 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:16 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:53:17.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:16 smithi045 ceph-mon[23737]: Detected new or changed devices on smithi165 2024-09-17T11:53:17.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:16 smithi045 ceph-mon[23737]: Adjusting osd_memory_target on smithi165 to 7968M 2024-09-17T11:53:17.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:16 smithi045 ceph-mon[23737]: osdmap e43: 6 total, 6 up, 6 in 2024-09-17T11:53:17.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:16 smithi045 ceph-mon[23737]: from='client.24189 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi165:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:53:17.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:16 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T11:53:17.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:16 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T11:53:17.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:16 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:53:17.901 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:17 smithi165 ceph-mon[25702]: pgmap v134: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T11:53:18.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:17 smithi045 ceph-mon[23737]: pgmap v134: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T11:53:19.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:18 smithi165 ceph-mon[25702]: from='client.? 172.21.15.165:0/4017224639' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7ba074c1-5f25-4279-bee7-5a8cccf7df4b"}]: dispatch 2024-09-17T11:53:19.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:18 smithi165 ceph-mon[25702]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7ba074c1-5f25-4279-bee7-5a8cccf7df4b"}]: dispatch 2024-09-17T11:53:19.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:18 smithi165 ceph-mon[25702]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7ba074c1-5f25-4279-bee7-5a8cccf7df4b"}]': finished 2024-09-17T11:53:19.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:18 smithi165 ceph-mon[25702]: osdmap e44: 7 total, 6 up, 7 in 2024-09-17T11:53:19.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:18 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T11:53:19.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:18 smithi045 ceph-mon[23737]: from='client.? 172.21.15.165:0/4017224639' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7ba074c1-5f25-4279-bee7-5a8cccf7df4b"}]: dispatch 2024-09-17T11:53:19.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:18 smithi045 ceph-mon[23737]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7ba074c1-5f25-4279-bee7-5a8cccf7df4b"}]: dispatch 2024-09-17T11:53:19.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:18 smithi045 ceph-mon[23737]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7ba074c1-5f25-4279-bee7-5a8cccf7df4b"}]': finished 2024-09-17T11:53:19.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:18 smithi045 ceph-mon[23737]: osdmap e44: 7 total, 6 up, 7 in 2024-09-17T11:53:19.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:18 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T11:53:20.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:19 smithi165 ceph-mon[25702]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-09-17T11:53:20.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:19 smithi165 ceph-mon[25702]: from='client.? 172.21.15.165:0/2662407011' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T11:53:20.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:19 smithi165 ceph-mon[25702]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T11:53:20.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:19 smithi165 ceph-mon[25702]: Cluster is now healthy 2024-09-17T11:53:20.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:19 smithi045 ceph-mon[23737]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-09-17T11:53:20.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:19 smithi045 ceph-mon[23737]: from='client.? 172.21.15.165:0/2662407011' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T11:53:20.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:19 smithi045 ceph-mon[23737]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T11:53:20.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:19 smithi045 ceph-mon[23737]: Cluster is now healthy 2024-09-17T11:53:21.925 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:21 smithi165 ceph-mon[25702]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T11:53:22.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:21 smithi045 ceph-mon[23737]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T11:53:23.915 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:23 smithi165 ceph-mon[25702]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-17T11:53:24.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:23 smithi045 ceph-mon[23737]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-17T11:53:25.011 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:24 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T11:53:25.011 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:24 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:53:25.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:24 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T11:53:25.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:24 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:53:25.998 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:25 smithi165 ceph-mon[25702]: Deploying daemon osd.6 on smithi165 2024-09-17T11:53:25.999 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:25 smithi165 ceph-mon[25702]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-17T11:53:26.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:25 smithi045 ceph-mon[23737]: Deploying daemon osd.6 on smithi165 2024-09-17T11:53:26.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:25 smithi045 ceph-mon[23737]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-17T11:53:28.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:27 smithi165 ceph-mon[25702]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T11:53:28.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:27 smithi045 ceph-mon[23737]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T11:53:29.752 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:29 smithi165 ceph-mon[25702]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T11:53:30.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:29 smithi045 ceph-mon[23737]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T11:53:31.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:30 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:31.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:30 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:53:31.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:30 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:53:31.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:30 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:53:31.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:30 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:31.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:30 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:53:31.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:30 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:53:31.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:30 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:53:32.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:31 smithi165 ceph-mon[25702]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T11:53:32.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:31 smithi045 ceph-mon[23737]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T11:53:32.759 INFO:teuthology.orchestra.run.smithi165.stdout:Created osd(s) 6 on host 'smithi165' 2024-09-17T11:53:33.357 DEBUG:teuthology.orchestra.run.smithi165:osd.6> sudo journalctl -f -n 0 -u ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.6.service 2024-09-17T11:53:33.359 INFO:tasks.cephadm:Deploying osd.7 on smithi165 with /dev/vg_nvme/lv_1... 2024-09-17T11:53:33.359 DEBUG:teuthology.orchestra.run.smithi165:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-17T11:53:33.940 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:33 smithi165 ceph-mon[25702]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T11:53:33.940 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:33 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:33.940 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:33 smithi165 ceph-mon[25702]: from='osd.6 [v2:172.21.15.165:6816/1036317166,v1:172.21.15.165:6817/1036317166]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T11:53:33.940 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:33 smithi165 ceph-mon[25702]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T11:53:34.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:33 smithi045 ceph-mon[23737]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T11:53:34.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:33 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:34.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:33 smithi045 ceph-mon[23737]: from='osd.6 [v2:172.21.15.165:6816/1036317166,v1:172.21.15.165:6817/1036317166]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T11:53:34.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:33 smithi045 ceph-mon[23737]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T11:53:35.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:34 smithi165 ceph-mon[25702]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T11:53:35.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:34 smithi165 ceph-mon[25702]: osdmap e45: 7 total, 6 up, 7 in 2024-09-17T11:53:35.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:34 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T11:53:35.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:34 smithi165 ceph-mon[25702]: from='osd.6 [v2:172.21.15.165:6816/1036317166,v1:172.21.15.165:6817/1036317166]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]: dispatch 2024-09-17T11:53:35.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:34 smithi165 ceph-mon[25702]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]: dispatch 2024-09-17T11:53:35.016 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 11:53:34 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[35853]: 2024-09-17T11:53:34.774+0000 7f5d5789a700 -1 osd.6 0 waiting for initial osdmap 2024-09-17T11:53:35.016 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 11:53:34 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[35853]: 2024-09-17T11:53:34.781+0000 7f5d52a32700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T11:53:35.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:34 smithi045 ceph-mon[23737]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T11:53:35.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:34 smithi045 ceph-mon[23737]: osdmap e45: 7 total, 6 up, 7 in 2024-09-17T11:53:35.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:34 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T11:53:35.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:34 smithi045 ceph-mon[23737]: from='osd.6 [v2:172.21.15.165:6816/1036317166,v1:172.21.15.165:6817/1036317166]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]: dispatch 2024-09-17T11:53:35.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:34 smithi045 ceph-mon[23737]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]: dispatch 2024-09-17T11:53:35.870 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:35 smithi165 ceph-mon[25702]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T11:53:35.871 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:35 smithi165 ceph-mon[25702]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]': finished 2024-09-17T11:53:35.871 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:35 smithi165 ceph-mon[25702]: osdmap e46: 7 total, 6 up, 7 in 2024-09-17T11:53:35.871 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:35 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T11:53:35.871 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:35 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T11:53:35.871 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:35 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:36.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:35 smithi045 ceph-mon[23737]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T11:53:36.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:35 smithi045 ceph-mon[23737]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]': finished 2024-09-17T11:53:36.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:35 smithi045 ceph-mon[23737]: osdmap e46: 7 total, 6 up, 7 in 2024-09-17T11:53:36.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:35 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T11:53:36.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:35 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T11:53:36.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:35 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:36.695 INFO:teuthology.orchestra.run.smithi165.stdout: 2024-09-17T11:53:36.714 DEBUG:teuthology.orchestra.run.smithi165:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi165:vg_nvme/lv_1 2024-09-17T11:53:36.998 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:36 smithi165 ceph-mon[25702]: purged_snaps scrub starts 2024-09-17T11:53:36.998 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:36 smithi165 ceph-mon[25702]: purged_snaps scrub ok 2024-09-17T11:53:36.999 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:36 smithi165 ceph-mon[25702]: osd.6 [v2:172.21.15.165:6816/1036317166,v1:172.21.15.165:6817/1036317166] boot 2024-09-17T11:53:36.999 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:36 smithi165 ceph-mon[25702]: osdmap e47: 7 total, 7 up, 7 in 2024-09-17T11:53:36.999 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:36 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T11:53:37.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:36 smithi045 ceph-mon[23737]: purged_snaps scrub starts 2024-09-17T11:53:37.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:36 smithi045 ceph-mon[23737]: purged_snaps scrub ok 2024-09-17T11:53:37.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:36 smithi045 ceph-mon[23737]: osd.6 [v2:172.21.15.165:6816/1036317166,v1:172.21.15.165:6817/1036317166] boot 2024-09-17T11:53:37.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:36 smithi045 ceph-mon[23737]: osdmap e47: 7 total, 7 up, 7 in 2024-09-17T11:53:37.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:36 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T11:53:38.001 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:37 smithi165 ceph-mon[25702]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-17T11:53:38.001 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:37 smithi165 ceph-mon[25702]: osdmap e48: 7 total, 7 up, 7 in 2024-09-17T11:53:38.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:37 smithi045 ceph-mon[23737]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-17T11:53:38.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:37 smithi045 ceph-mon[23737]: osdmap e48: 7 total, 7 up, 7 in 2024-09-17T11:53:39.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:38 smithi165 ceph-mon[25702]: osdmap e49: 7 total, 7 up, 7 in 2024-09-17T11:53:39.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:38 smithi045 ceph-mon[23737]: osdmap e49: 7 total, 7 up, 7 in 2024-09-17T11:53:40.080 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:39 smithi165 ceph-mon[25702]: pgmap v151: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-09-17T11:53:40.080 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:39 smithi165 ceph-mon[25702]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-09-17T11:53:40.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:39 smithi045 ceph-mon[23737]: pgmap v151: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-09-17T11:53:40.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:39 smithi045 ceph-mon[23737]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2024-09-17T11:53:41.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:40 smithi165 ceph-mon[25702]: osdmap e50: 7 total, 7 up, 7 in 2024-09-17T11:53:41.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:40 smithi165 ceph-mon[25702]: from='client.24207 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi165:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:53:41.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:40 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T11:53:41.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:40 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T11:53:41.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:40 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:53:41.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:40 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:41.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:40 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:53:41.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:40 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:53:41.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:40 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:53:41.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:40 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:41.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:40 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:41.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:40 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:53:41.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:40 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:53:41.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:40 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:53:41.059 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:40 smithi045 ceph-mon[23737]: osdmap e50: 7 total, 7 up, 7 in 2024-09-17T11:53:41.059 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:40 smithi045 ceph-mon[23737]: from='client.24207 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi165:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:53:41.059 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:40 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T11:53:41.059 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:40 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T11:53:41.060 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:40 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:53:41.060 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:40 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:41.060 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:40 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:53:41.060 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:40 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:53:41.060 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:40 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:53:41.060 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:40 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:41.060 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:40 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:41.060 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:40 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:53:41.060 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:40 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:53:41.060 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:40 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:53:41.959 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:41 smithi165 ceph-mon[25702]: pgmap v153: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-09-17T11:53:41.959 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:41 smithi165 ceph-mon[25702]: Detected new or changed devices on smithi165 2024-09-17T11:53:41.959 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:41 smithi165 ceph-mon[25702]: Adjusting osd_memory_target on smithi165 to 5312M 2024-09-17T11:53:41.959 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:41 smithi165 ceph-mon[25702]: osdmap e51: 7 total, 7 up, 7 in 2024-09-17T11:53:41.959 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:41 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:42.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:41 smithi045 ceph-mon[23737]: pgmap v153: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 2/6 objects degraded (33.333%) 2024-09-17T11:53:42.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:41 smithi045 ceph-mon[23737]: Detected new or changed devices on smithi165 2024-09-17T11:53:42.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:41 smithi045 ceph-mon[23737]: Adjusting osd_memory_target on smithi165 to 5312M 2024-09-17T11:53:42.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:41 smithi045 ceph-mon[23737]: osdmap e51: 7 total, 7 up, 7 in 2024-09-17T11:53:42.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:41 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:44.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:43 smithi165 ceph-mon[25702]: pgmap v155: 1 pgs: 1 peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T11:53:44.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:43 smithi165 ceph-mon[25702]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-09-17T11:53:44.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:43 smithi165 ceph-mon[25702]: Cluster is now healthy 2024-09-17T11:53:44.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:43 smithi165 ceph-mon[25702]: from='client.? 172.21.15.165:0/1309520949' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9be76121-235e-46bc-8a13-44e2a409b03b"}]: dispatch 2024-09-17T11:53:44.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:43 smithi165 ceph-mon[25702]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9be76121-235e-46bc-8a13-44e2a409b03b"}]: dispatch 2024-09-17T11:53:44.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:43 smithi165 ceph-mon[25702]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9be76121-235e-46bc-8a13-44e2a409b03b"}]': finished 2024-09-17T11:53:44.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:43 smithi165 ceph-mon[25702]: osdmap e52: 8 total, 7 up, 8 in 2024-09-17T11:53:44.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T11:53:44.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:43 smithi165 ceph-mon[25702]: from='client.? 172.21.15.165:0/2227997812' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T11:53:44.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:43 smithi045 ceph-mon[23737]: pgmap v155: 1 pgs: 1 peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T11:53:44.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:43 smithi045 ceph-mon[23737]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2024-09-17T11:53:44.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:43 smithi045 ceph-mon[23737]: Cluster is now healthy 2024-09-17T11:53:44.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:43 smithi045 ceph-mon[23737]: from='client.? 172.21.15.165:0/1309520949' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9be76121-235e-46bc-8a13-44e2a409b03b"}]: dispatch 2024-09-17T11:53:44.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:43 smithi045 ceph-mon[23737]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9be76121-235e-46bc-8a13-44e2a409b03b"}]: dispatch 2024-09-17T11:53:44.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:43 smithi045 ceph-mon[23737]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9be76121-235e-46bc-8a13-44e2a409b03b"}]': finished 2024-09-17T11:53:44.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:43 smithi045 ceph-mon[23737]: osdmap e52: 8 total, 7 up, 8 in 2024-09-17T11:53:44.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T11:53:44.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:43 smithi045 ceph-mon[23737]: from='client.? 172.21.15.165:0/2227997812' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T11:53:46.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:45 smithi165 ceph-mon[25702]: pgmap v157: 1 pgs: 1 peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T11:53:46.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:45 smithi045 ceph-mon[23737]: pgmap v157: 1 pgs: 1 peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T11:53:47.980 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:47 smithi165 ceph-mon[25702]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-09-17T11:53:48.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:47 smithi045 ceph-mon[23737]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-09-17T11:53:49.823 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:49 smithi165 ceph-mon[25702]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-17T11:53:49.823 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:49 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T11:53:49.823 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:49 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:53:49.823 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:49 smithi165 ceph-mon[25702]: Deploying daemon osd.7 on smithi165 2024-09-17T11:53:50.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:49 smithi045 ceph-mon[23737]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-17T11:53:50.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:49 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T11:53:50.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:49 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:53:50.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:49 smithi045 ceph-mon[23737]: Deploying daemon osd.7 on smithi165 2024-09-17T11:53:51.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:50 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:53:51.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:50 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:53:51.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:50 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:53:51.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:50 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:53:51.834 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:51 smithi165 ceph-mon[25702]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-09-17T11:53:52.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:51 smithi045 ceph-mon[23737]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-09-17T11:53:53.966 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:53 smithi165 ceph-mon[25702]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T11:53:54.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:53 smithi045 ceph-mon[23737]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T11:53:55.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:55 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:55.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:55 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:53:55.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:55 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:53:55.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:55 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:53:55.431 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:55 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:55.431 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:55 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:53:55.432 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:55 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:53:55.432 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:55 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:53:56.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:56 smithi165 ceph-mon[25702]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 44 KiB/s, 0 objects/s recovering 2024-09-17T11:53:56.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:56 smithi045 ceph-mon[23737]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 44 KiB/s, 0 objects/s recovering 2024-09-17T11:53:57.533 INFO:teuthology.orchestra.run.smithi165.stdout:Created osd(s) 7 on host 'smithi165' 2024-09-17T11:53:58.214 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:58 smithi165 ceph-mon[25702]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-17T11:53:58.214 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:58 smithi165 ceph-mon[25702]: from='osd.7 [v2:172.21.15.165:6824/446054329,v1:172.21.15.165:6825/446054329]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T11:53:58.214 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:58 smithi165 ceph-mon[25702]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T11:53:58.214 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:58 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:58.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:58 smithi045 ceph-mon[23737]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-17T11:53:58.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:58 smithi045 ceph-mon[23737]: from='osd.7 [v2:172.21.15.165:6824/446054329,v1:172.21.15.165:6825/446054329]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T11:53:58.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:58 smithi045 ceph-mon[23737]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T11:53:58.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:58 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:58.478 DEBUG:teuthology.orchestra.run.smithi165:osd.7> sudo journalctl -f -n 0 -u ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.7.service 2024-09-17T11:53:58.480 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-17T11:53:58.480 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-17T11:53:59.049 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:59 smithi045 ceph-mon[23737]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T11:53:59.049 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:59 smithi045 ceph-mon[23737]: osdmap e53: 8 total, 7 up, 8 in 2024-09-17T11:53:59.049 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:59 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T11:53:59.049 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:59 smithi045 ceph-mon[23737]: from='osd.7 [v2:172.21.15.165:6824/446054329,v1:172.21.15.165:6825/446054329]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]: dispatch 2024-09-17T11:53:59.049 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:59 smithi045 ceph-mon[23737]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]: dispatch 2024-09-17T11:53:59.049 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:53:59 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:59.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:59 smithi165 ceph-mon[25702]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T11:53:59.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:59 smithi165 ceph-mon[25702]: osdmap e53: 8 total, 7 up, 8 in 2024-09-17T11:53:59.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:59 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T11:53:59.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:59 smithi165 ceph-mon[25702]: from='osd.7 [v2:172.21.15.165:6824/446054329,v1:172.21.15.165:6825/446054329]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]: dispatch 2024-09-17T11:53:59.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:59 smithi165 ceph-mon[25702]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]: dispatch 2024-09-17T11:53:59.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:53:59 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:53:59.266 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 11:53:59 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[38891]: 2024-09-17T11:53:59.037+0000 7f49df239700 -1 osd.7 0 waiting for initial osdmap 2024-09-17T11:53:59.266 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 11:53:59 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[38891]: 2024-09-17T11:53:59.045+0000 7f49db3d3700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T11:54:00.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:00 smithi045 ceph-mon[23737]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T11:54:00.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:00 smithi045 ceph-mon[23737]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]': finished 2024-09-17T11:54:00.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:00 smithi045 ceph-mon[23737]: osdmap e54: 8 total, 7 up, 8 in 2024-09-17T11:54:00.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:00 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T11:54:00.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:00 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T11:54:00.258 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:00 smithi165 ceph-mon[25702]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T11:54:00.258 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:00 smithi165 ceph-mon[25702]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi165", "root=default"]}]': finished 2024-09-17T11:54:00.258 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:00 smithi165 ceph-mon[25702]: osdmap e54: 8 total, 7 up, 8 in 2024-09-17T11:54:00.258 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:00 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T11:54:00.258 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:00 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T11:54:00.729 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:54:01.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:01 smithi045 ceph-mon[23737]: purged_snaps scrub starts 2024-09-17T11:54:01.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:01 smithi045 ceph-mon[23737]: purged_snaps scrub ok 2024-09-17T11:54:01.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:01 smithi045 ceph-mon[23737]: osd.7 [v2:172.21.15.165:6824/446054329,v1:172.21.15.165:6825/446054329] boot 2024-09-17T11:54:01.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:01 smithi045 ceph-mon[23737]: osdmap e55: 8 total, 8 up, 8 in 2024-09-17T11:54:01.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:01 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T11:54:01.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:01 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/2786954912' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-17T11:54:01.477 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:01 smithi165 ceph-mon[25702]: purged_snaps scrub starts 2024-09-17T11:54:01.478 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:01 smithi165 ceph-mon[25702]: purged_snaps scrub ok 2024-09-17T11:54:01.478 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:01 smithi165 ceph-mon[25702]: osd.7 [v2:172.21.15.165:6824/446054329,v1:172.21.15.165:6825/446054329] boot 2024-09-17T11:54:01.478 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:01 smithi165 ceph-mon[25702]: osdmap e55: 8 total, 8 up, 8 in 2024-09-17T11:54:01.478 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:01 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T11:54:01.478 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:01 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/2786954912' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-17T11:54:01.515 INFO:teuthology.orchestra.run.smithi045.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1726574040,"num_in_osds":8,"osd_in_since":1726574022,"num_remapped_pgs":1} 2024-09-17T11:54:01.515 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-17T11:54:01.738 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:54:02.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:02 smithi045 ceph-mon[23737]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:02.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:02 smithi045 ceph-mon[23737]: osdmap e56: 8 total, 8 up, 8 in 2024-09-17T11:54:02.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:02 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:54:02.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:02 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:54:02.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:02 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:54:02.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:02 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:54:02.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:02 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:54:02.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:02 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:54:02.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:02 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:54:02.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:02 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:54:02.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:02 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:54:02.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:02 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:54:02.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:02 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:54:02.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:02 smithi165 ceph-mon[25702]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:02.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:02 smithi165 ceph-mon[25702]: osdmap e56: 8 total, 8 up, 8 in 2024-09-17T11:54:02.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:02 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:54:02.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:02 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:54:02.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:02 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:54:02.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:02 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:54:02.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:02 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:54:02.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:02 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:54:02.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:02 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:54:02.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:02 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:54:02.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:02 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:54:02.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:02 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:54:02.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:02 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:54:03.179 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:03 smithi045 ceph-mon[23737]: Detected new or changed devices on smithi165 2024-09-17T11:54:03.179 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:03 smithi045 ceph-mon[23737]: Adjusting osd_memory_target on smithi165 to 3984M 2024-09-17T11:54:03.179 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:03 smithi045 ceph-mon[23737]: osdmap e57: 8 total, 8 up, 8 in 2024-09-17T11:54:03.179 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:03 smithi045 ceph-mon[23737]: osdmap e58: 8 total, 8 up, 8 in 2024-09-17T11:54:03.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:03 smithi165 ceph-mon[25702]: Detected new or changed devices on smithi165 2024-09-17T11:54:03.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:03 smithi165 ceph-mon[25702]: Adjusting osd_memory_target on smithi165 to 3984M 2024-09-17T11:54:03.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:03 smithi165 ceph-mon[25702]: osdmap e57: 8 total, 8 up, 8 in 2024-09-17T11:54:03.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:03 smithi165 ceph-mon[25702]: osdmap e58: 8 total, 8 up, 8 in 2024-09-17T11:54:03.612 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:54:03.612 INFO:teuthology.orchestra.run.smithi045.stdout:{"epoch":58,"fsid":"a12537de-74ea-11ef-bceb-c7b262605968","created":"2024-09-17T11:48:03.165915+0000","modified":"2024-09-17T11:54:02.702225+0000","last_up_change":"2024-09-17T11:54:00.032170+0000","last_in_change":"2024-09-17T11:53:42.818837+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-17T11:51:57.107669+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"e288efb6-6c20-40b0-95fd-75180fb6f1c8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":3351925855},{"type":"v1","addr":"172.21.15.45:6803","nonce":3351925855}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":3351925855},{"type":"v1","addr":"172.21.15.45:6805","nonce":3351925855}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":3351925855},{"type":"v1","addr":"172.21.15.45:6809","nonce":3351925855}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":3351925855},{"type":"v1","addr":"172.21.15.45:6807","nonce":3351925855}]},"public_addr":"172.21.15.45:6803/3351925855","cluster_addr":"172.21.15.45:6805/3351925855","heartbeat_back_addr":"172.21.15.45:6809/3351925855","heartbeat_front_addr":"172.21.15.45:6807/3351925855","state":["exists","up"]},{"osd":1,"uuid":"c063cd08-adc2-4ad1-a307-1fb7b3d2b5f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2799716519},{"type":"v1","addr":"172.21.15.45:6811","nonce":2799716519}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2799716519},{"type":"v1","addr":"172.21.15.45:6813","nonce":2799716519}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2799716519},{"type":"v1","addr":"172.21.15.45:6817","nonce":2799716519}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2799716519},{"type":"v1","addr":"172.21.15.45:6815","nonce":2799716519}]},"public_addr":"172.21.15.45:6811/2799716519","cluster_addr":"172.21.15.45:6813/2799716519","heartbeat_back_addr":"172.21.15.45:6817/2799716519","heartbeat_front_addr":"172.21.15.45:6815/2799716519","state":["exists","up"]},{"osd":2,"uuid":"ea51869a-b1ac-483b-8f4e-8d14dc4093ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2178441384},{"type":"v1","addr":"172.21.15.45:6819","nonce":2178441384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2178441384},{"type":"v1","addr":"172.21.15.45:6821","nonce":2178441384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":2178441384},{"type":"v1","addr":"172.21.15.45:6825","nonce":2178441384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2178441384},{"type":"v1","addr":"172.21.15.45:6823","nonce":2178441384}]},"public_addr":"172.21.15.45:6819/2178441384","cluster_addr":"172.21.15.45:6821/2178441384","heartbeat_back_addr":"172.21.15.45:6825/2178441384","heartbeat_front_addr":"172.21.15.45:6823/2178441384","state":["exists","up"]},{"osd":3,"uuid":"fbc72a30-0ec4-4738-86fb-b0ec8993e695","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":2915608972},{"type":"v1","addr":"172.21.15.45:6827","nonce":2915608972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":2915608972},{"type":"v1","addr":"172.21.15.45:6829","nonce":2915608972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2915608972},{"type":"v1","addr":"172.21.15.45:6833","nonce":2915608972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":2915608972},{"type":"v1","addr":"172.21.15.45:6831","nonce":2915608972}]},"public_addr":"172.21.15.45:6827/2915608972","cluster_addr":"172.21.15.45:6829/2915608972","heartbeat_back_addr":"172.21.15.45:6833/2915608972","heartbeat_front_addr":"172.21.15.45:6831/2915608972","state":["exists","up"]},{"osd":4,"uuid":"e538740a-c502-4d61-83b2-1847a9083ff5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6800","nonce":2995343118},{"type":"v1","addr":"172.21.15.165:6801","nonce":2995343118}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6802","nonce":2995343118},{"type":"v1","addr":"172.21.15.165:6803","nonce":2995343118}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6806","nonce":2995343118},{"type":"v1","addr":"172.21.15.165:6807","nonce":2995343118}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6804","nonce":2995343118},{"type":"v1","addr":"172.21.15.165:6805","nonce":2995343118}]},"public_addr":"172.21.15.165:6801/2995343118","cluster_addr":"172.21.15.165:6803/2995343118","heartbeat_back_addr":"172.21.15.165:6807/2995343118","heartbeat_front_addr":"172.21.15.165:6805/2995343118","state":["exists","up"]},{"osd":5,"uuid":"20205cf8-a248-4b9d-9e97-c43f8ded5924","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6808","nonce":2948491063},{"type":"v1","addr":"172.21.15.165:6809","nonce":2948491063}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6810","nonce":2948491063},{"type":"v1","addr":"172.21.15.165:6811","nonce":2948491063}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6814","nonce":2948491063},{"type":"v1","addr":"172.21.15.165:6815","nonce":2948491063}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6812","nonce":2948491063},{"type":"v1","addr":"172.21.15.165:6813","nonce":2948491063}]},"public_addr":"172.21.15.165:6809/2948491063","cluster_addr":"172.21.15.165:6811/2948491063","heartbeat_back_addr":"172.21.15.165:6815/2948491063","heartbeat_front_addr":"172.21.15.165:6813/2948491063","state":["exists","up"]},{"osd":6,"uuid":"7ba074c1-5f25-4279-bee7-5a8cccf7df4b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6816","nonce":1036317166},{"type":"v1","addr":"172.21.15.165:6817","nonce":1036317166}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6818","nonce":1036317166},{"type":"v1","addr":"172.21.15.165:6819","nonce":1036317166}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6822","nonce":1036317166},{"type":"v1","addr":"172.21.15.165:6823","nonce":1036317166}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6820","nonce":1036317166},{"type":"v1","addr":"172.21.15.165:6821","nonce":1036317166}]},"public_addr":"172.21.15.165:6817/1036317166","cluster_addr":"172.21.15.165:6819/1036317166","heartbeat_back_addr":"172.21.15.165:6823/1036317166","heartbeat_front_addr":"172.21.15.165:6821/1036317166","state":["exists","up"]},{"osd":7,"uuid":"9be76121-235e-46bc-8a13-44e2a409b03b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6824","nonce":446054329},{"type":"v1","addr":"172.21.15.165:6825","nonce":446054329}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6826","nonce":446054329},{"type":"v1","addr":"172.21.15.165:6827","nonce":446054329}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6830","nonce":446054329},{"type":"v1","addr":"172.21.15.165:6831","nonce":446054329}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6828","nonce":446054329},{"type":"v1","addr":"172.21.15.165:6829","nonce":446054329}]},"public_addr":"172.21.15.165:6825/446054329","cluster_addr":"172.21.15.165:6827/446054329","heartbeat_back_addr":"172.21.15.165:6831/446054329","heartbeat_front_addr":"172.21.15.165:6829/446054329","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:50:59.927811+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:51:26.369952+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:51:53.856372+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:52:23.587819+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:52:46.069345+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:53:10.166892+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:53:33.859394+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:53:58.133456+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.45:0/853583807":"2024-09-18T11:49:50.540848+0000","172.21.15.45:6801/2165128626":"2024-09-18T11:49:02.452594+0000","172.21.15.45:0/3179235283":"2024-09-18T11:49:50.540848+0000","172.21.15.45:0/1137733817":"2024-09-18T11:49:50.540848+0000","172.21.15.45:0/399753365":"2024-09-18T11:49:02.452594+0000","172.21.15.45:0/469356163":"2024-09-18T11:49:02.452594+0000","172.21.15.45:0/2062074897":"2024-09-18T11:49:02.452594+0000","172.21.15.45:6801/881200176":"2024-09-18T11:49:50.540848+0000","172.21.15.45:0/1225275789":"2024-09-18T11:48:32.377129+0000","172.21.15.45:6800/436731029":"2024-09-18T11:48:32.377129+0000","172.21.15.45:6801/436731029":"2024-09-18T11:48:32.377129+0000","172.21.15.45:6800/881200176":"2024-09-18T11:49:50.540848+0000","172.21.15.45:0/2821439202":"2024-09-18T11:48:32.377129+0000","172.21.15.45:6800/2165128626":"2024-09-18T11:49:02.452594+0000","172.21.15.45:0/515207401":"2024-09-18T11:48:32.377129+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T11:54:04.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:04 smithi045 ceph-mon[23737]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:04.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:04 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/3551504377' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T11:54:04.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:04 smithi045 ceph-mon[23737]: osdmap e59: 8 total, 8 up, 8 in 2024-09-17T11:54:04.407 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-17T11:51:57.107669+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-17T11:54:04.408 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-17T11:54:04.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:04 smithi165 ceph-mon[25702]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:04.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:04 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/3551504377' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T11:54:04.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:04 smithi165 ceph-mon[25702]: osdmap e59: 8 total, 8 up, 8 in 2024-09-17T11:54:04.633 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:54:06.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:06 smithi045 ceph-mon[23737]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:06.431 INFO:teuthology.orchestra.run.smithi045.stdout:pg_num: 1 2024-09-17T11:54:06.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:06 smithi165 ceph-mon[25702]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:07.069 INFO:tasks.cephadm:Setting up client nodes... 2024-09-17T11:54:07.069 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-17T11:54:07.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:07 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/2525614443' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-17T11:54:07.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:07 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/2525614443' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-17T11:54:08.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:08 smithi045 ceph-mon[23737]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-09-17T11:54:08.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:08 smithi165 ceph-mon[25702]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-09-17T11:54:09.263 INFO:teuthology.orchestra.run.smithi045.stdout:[client.0] 2024-09-17T11:54:09.263 INFO:teuthology.orchestra.run.smithi045.stdout: key = AQDhbelmBQxiDxAAylumfQHfVym+bjv0N6LdjQ== 2024-09-17T11:54:09.973 DEBUG:teuthology.orchestra.run.smithi045:> set -ex 2024-09-17T11:54:09.973 DEBUG:teuthology.orchestra.run.smithi045:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-17T11:54:09.973 DEBUG:teuthology.orchestra.run.smithi045:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-17T11:54:10.013 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph config log 1 --format=json 2024-09-17T11:54:10.265 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:54:10.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:10 smithi045 ceph-mon[23737]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-09-17T11:54:10.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:10 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/2107727969' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T11:54:10.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:10 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/2107727969' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-17T11:54:10.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:10 smithi165 ceph-mon[25702]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-09-17T11:54:10.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:10 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/2107727969' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T11:54:10.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:10 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/2107727969' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-17T11:54:12.102 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:54:12.317 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:12 smithi165 ceph-mon[25702]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-17T11:54:12.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:12 smithi045 ceph-mon[23737]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-17T11:54:12.754 INFO:teuthology.orchestra.run.smithi045.stdout:[{"version":20,"timestamp":"2024-09-17T11:54:01.778094+0000","name":"","changes":[{"name":"osd/host:smithi165/osd_memory_target","previous_value":"5570076945","new_value":"4177557708"}]}] 2024-09-17T11:54:12.754 INFO:tasks.ceph_manager:config epoch is 20 2024-09-17T11:54:12.754 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-17T11:54:12.754 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-17T11:54:12.755 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-17T11:54:12.971 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:54:13.325 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:13 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/2811283140' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-17T11:54:13.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:13 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/2811283140' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-17T11:54:14.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:14 smithi045 ceph-mon[23737]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T11:54:14.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:14 smithi165 ceph-mon[25702]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T11:54:14.977 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:54:15.335 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:15 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/2914371801' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T11:54:15.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:15 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/2914371801' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T11:54:15.801 INFO:teuthology.orchestra.run.smithi045.stdout:{"epoch":19,"active_gid":14200,"active_name":"smithi045.jtwyrp","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6800","nonce":1851578750},{"type":"v1","addr":"172.21.15.45:6801","nonce":1851578750}]},"active_addr":"172.21.15.45:6801/1851578750","active_change":"2024-09-17T11:49:50.541208+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14218,"name":"smithi165.ykdszc","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.45:8443/","prometheus":"http://172.21.15.45:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.45:0","nonce":900590788}]},{"addrvec":[{"type":"v2","addr":"172.21.15.45:0","nonce":448024149}]},{"addrvec":[{"type":"v2","addr":"172.21.15.45:0","nonce":3879686899}]},{"addrvec":[{"type":"v2","addr":"172.21.15.45:0","nonce":4200294376}]}]}} 2024-09-17T11:54:15.854 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-17T11:54:15.854 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-17T11:54:15.854 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-17T11:54:16.082 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:54:16.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:16 smithi045 ceph-mon[23737]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s, 0 objects/s recovering 2024-09-17T11:54:16.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:16 smithi165 ceph-mon[25702]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s, 0 objects/s recovering 2024-09-17T11:54:18.089 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:54:18.090 INFO:teuthology.orchestra.run.smithi045.stdout:{"epoch":59,"fsid":"a12537de-74ea-11ef-bceb-c7b262605968","created":"2024-09-17T11:48:03.165915+0000","modified":"2024-09-17T11:54:03.707875+0000","last_up_change":"2024-09-17T11:54:00.032170+0000","last_in_change":"2024-09-17T11:53:42.818837+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-17T11:51:57.107669+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"e288efb6-6c20-40b0-95fd-75180fb6f1c8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":3351925855},{"type":"v1","addr":"172.21.15.45:6803","nonce":3351925855}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":3351925855},{"type":"v1","addr":"172.21.15.45:6805","nonce":3351925855}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":3351925855},{"type":"v1","addr":"172.21.15.45:6809","nonce":3351925855}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":3351925855},{"type":"v1","addr":"172.21.15.45:6807","nonce":3351925855}]},"public_addr":"172.21.15.45:6803/3351925855","cluster_addr":"172.21.15.45:6805/3351925855","heartbeat_back_addr":"172.21.15.45:6809/3351925855","heartbeat_front_addr":"172.21.15.45:6807/3351925855","state":["exists","up"]},{"osd":1,"uuid":"c063cd08-adc2-4ad1-a307-1fb7b3d2b5f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2799716519},{"type":"v1","addr":"172.21.15.45:6811","nonce":2799716519}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2799716519},{"type":"v1","addr":"172.21.15.45:6813","nonce":2799716519}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2799716519},{"type":"v1","addr":"172.21.15.45:6817","nonce":2799716519}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2799716519},{"type":"v1","addr":"172.21.15.45:6815","nonce":2799716519}]},"public_addr":"172.21.15.45:6811/2799716519","cluster_addr":"172.21.15.45:6813/2799716519","heartbeat_back_addr":"172.21.15.45:6817/2799716519","heartbeat_front_addr":"172.21.15.45:6815/2799716519","state":["exists","up"]},{"osd":2,"uuid":"ea51869a-b1ac-483b-8f4e-8d14dc4093ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2178441384},{"type":"v1","addr":"172.21.15.45:6819","nonce":2178441384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2178441384},{"type":"v1","addr":"172.21.15.45:6821","nonce":2178441384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":2178441384},{"type":"v1","addr":"172.21.15.45:6825","nonce":2178441384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2178441384},{"type":"v1","addr":"172.21.15.45:6823","nonce":2178441384}]},"public_addr":"172.21.15.45:6819/2178441384","cluster_addr":"172.21.15.45:6821/2178441384","heartbeat_back_addr":"172.21.15.45:6825/2178441384","heartbeat_front_addr":"172.21.15.45:6823/2178441384","state":["exists","up"]},{"osd":3,"uuid":"fbc72a30-0ec4-4738-86fb-b0ec8993e695","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":2915608972},{"type":"v1","addr":"172.21.15.45:6827","nonce":2915608972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":2915608972},{"type":"v1","addr":"172.21.15.45:6829","nonce":2915608972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2915608972},{"type":"v1","addr":"172.21.15.45:6833","nonce":2915608972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":2915608972},{"type":"v1","addr":"172.21.15.45:6831","nonce":2915608972}]},"public_addr":"172.21.15.45:6827/2915608972","cluster_addr":"172.21.15.45:6829/2915608972","heartbeat_back_addr":"172.21.15.45:6833/2915608972","heartbeat_front_addr":"172.21.15.45:6831/2915608972","state":["exists","up"]},{"osd":4,"uuid":"e538740a-c502-4d61-83b2-1847a9083ff5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6800","nonce":2995343118},{"type":"v1","addr":"172.21.15.165:6801","nonce":2995343118}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6802","nonce":2995343118},{"type":"v1","addr":"172.21.15.165:6803","nonce":2995343118}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6806","nonce":2995343118},{"type":"v1","addr":"172.21.15.165:6807","nonce":2995343118}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6804","nonce":2995343118},{"type":"v1","addr":"172.21.15.165:6805","nonce":2995343118}]},"public_addr":"172.21.15.165:6801/2995343118","cluster_addr":"172.21.15.165:6803/2995343118","heartbeat_back_addr":"172.21.15.165:6807/2995343118","heartbeat_front_addr":"172.21.15.165:6805/2995343118","state":["exists","up"]},{"osd":5,"uuid":"20205cf8-a248-4b9d-9e97-c43f8ded5924","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6808","nonce":2948491063},{"type":"v1","addr":"172.21.15.165:6809","nonce":2948491063}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6810","nonce":2948491063},{"type":"v1","addr":"172.21.15.165:6811","nonce":2948491063}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6814","nonce":2948491063},{"type":"v1","addr":"172.21.15.165:6815","nonce":2948491063}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6812","nonce":2948491063},{"type":"v1","addr":"172.21.15.165:6813","nonce":2948491063}]},"public_addr":"172.21.15.165:6809/2948491063","cluster_addr":"172.21.15.165:6811/2948491063","heartbeat_back_addr":"172.21.15.165:6815/2948491063","heartbeat_front_addr":"172.21.15.165:6813/2948491063","state":["exists","up"]},{"osd":6,"uuid":"7ba074c1-5f25-4279-bee7-5a8cccf7df4b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6816","nonce":1036317166},{"type":"v1","addr":"172.21.15.165:6817","nonce":1036317166}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6818","nonce":1036317166},{"type":"v1","addr":"172.21.15.165:6819","nonce":1036317166}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6822","nonce":1036317166},{"type":"v1","addr":"172.21.15.165:6823","nonce":1036317166}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6820","nonce":1036317166},{"type":"v1","addr":"172.21.15.165:6821","nonce":1036317166}]},"public_addr":"172.21.15.165:6817/1036317166","cluster_addr":"172.21.15.165:6819/1036317166","heartbeat_back_addr":"172.21.15.165:6823/1036317166","heartbeat_front_addr":"172.21.15.165:6821/1036317166","state":["exists","up"]},{"osd":7,"uuid":"9be76121-235e-46bc-8a13-44e2a409b03b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6824","nonce":446054329},{"type":"v1","addr":"172.21.15.165:6825","nonce":446054329}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6826","nonce":446054329},{"type":"v1","addr":"172.21.15.165:6827","nonce":446054329}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6830","nonce":446054329},{"type":"v1","addr":"172.21.15.165:6831","nonce":446054329}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6828","nonce":446054329},{"type":"v1","addr":"172.21.15.165:6829","nonce":446054329}]},"public_addr":"172.21.15.165:6825/446054329","cluster_addr":"172.21.15.165:6827/446054329","heartbeat_back_addr":"172.21.15.165:6831/446054329","heartbeat_front_addr":"172.21.15.165:6829/446054329","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:50:59.927811+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:51:26.369952+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:51:53.856372+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:52:23.587819+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:52:46.069345+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:53:10.166892+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:53:33.859394+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:53:58.133456+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.45:0/853583807":"2024-09-18T11:49:50.540848+0000","172.21.15.45:6801/2165128626":"2024-09-18T11:49:02.452594+0000","172.21.15.45:0/3179235283":"2024-09-18T11:49:50.540848+0000","172.21.15.45:0/1137733817":"2024-09-18T11:49:50.540848+0000","172.21.15.45:0/399753365":"2024-09-18T11:49:02.452594+0000","172.21.15.45:0/469356163":"2024-09-18T11:49:02.452594+0000","172.21.15.45:0/2062074897":"2024-09-18T11:49:02.452594+0000","172.21.15.45:6801/881200176":"2024-09-18T11:49:50.540848+0000","172.21.15.45:0/1225275789":"2024-09-18T11:48:32.377129+0000","172.21.15.45:6800/436731029":"2024-09-18T11:48:32.377129+0000","172.21.15.45:6801/436731029":"2024-09-18T11:48:32.377129+0000","172.21.15.45:6800/881200176":"2024-09-18T11:49:50.540848+0000","172.21.15.45:0/2821439202":"2024-09-18T11:48:32.377129+0000","172.21.15.45:6800/2165128626":"2024-09-18T11:49:02.452594+0000","172.21.15.45:0/515207401":"2024-09-18T11:48:32.377129+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T11:54:18.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:18 smithi045 ceph-mon[23737]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-17T11:54:18.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:18 smithi165 ceph-mon[25702]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-17T11:54:18.830 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-17T11:54:18.830 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-17T11:54:19.049 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:54:19.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:19 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/1092965397' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T11:54:19.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:19 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/1092965397' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T11:54:20.185 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:20 smithi045 ceph-mon[23737]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:20.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:20 smithi165 ceph-mon[25702]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:20.867 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:54:20.867 INFO:teuthology.orchestra.run.smithi045.stdout:{"epoch":59,"fsid":"a12537de-74ea-11ef-bceb-c7b262605968","created":"2024-09-17T11:48:03.165915+0000","modified":"2024-09-17T11:54:03.707875+0000","last_up_change":"2024-09-17T11:54:00.032170+0000","last_in_change":"2024-09-17T11:53:42.818837+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-17T11:51:57.107669+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"e288efb6-6c20-40b0-95fd-75180fb6f1c8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6802","nonce":3351925855},{"type":"v1","addr":"172.21.15.45:6803","nonce":3351925855}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6804","nonce":3351925855},{"type":"v1","addr":"172.21.15.45:6805","nonce":3351925855}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6808","nonce":3351925855},{"type":"v1","addr":"172.21.15.45:6809","nonce":3351925855}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6806","nonce":3351925855},{"type":"v1","addr":"172.21.15.45:6807","nonce":3351925855}]},"public_addr":"172.21.15.45:6803/3351925855","cluster_addr":"172.21.15.45:6805/3351925855","heartbeat_back_addr":"172.21.15.45:6809/3351925855","heartbeat_front_addr":"172.21.15.45:6807/3351925855","state":["exists","up"]},{"osd":1,"uuid":"c063cd08-adc2-4ad1-a307-1fb7b3d2b5f3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6810","nonce":2799716519},{"type":"v1","addr":"172.21.15.45:6811","nonce":2799716519}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6812","nonce":2799716519},{"type":"v1","addr":"172.21.15.45:6813","nonce":2799716519}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6816","nonce":2799716519},{"type":"v1","addr":"172.21.15.45:6817","nonce":2799716519}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6814","nonce":2799716519},{"type":"v1","addr":"172.21.15.45:6815","nonce":2799716519}]},"public_addr":"172.21.15.45:6811/2799716519","cluster_addr":"172.21.15.45:6813/2799716519","heartbeat_back_addr":"172.21.15.45:6817/2799716519","heartbeat_front_addr":"172.21.15.45:6815/2799716519","state":["exists","up"]},{"osd":2,"uuid":"ea51869a-b1ac-483b-8f4e-8d14dc4093ad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6818","nonce":2178441384},{"type":"v1","addr":"172.21.15.45:6819","nonce":2178441384}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6820","nonce":2178441384},{"type":"v1","addr":"172.21.15.45:6821","nonce":2178441384}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6824","nonce":2178441384},{"type":"v1","addr":"172.21.15.45:6825","nonce":2178441384}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6822","nonce":2178441384},{"type":"v1","addr":"172.21.15.45:6823","nonce":2178441384}]},"public_addr":"172.21.15.45:6819/2178441384","cluster_addr":"172.21.15.45:6821/2178441384","heartbeat_back_addr":"172.21.15.45:6825/2178441384","heartbeat_front_addr":"172.21.15.45:6823/2178441384","state":["exists","up"]},{"osd":3,"uuid":"fbc72a30-0ec4-4738-86fb-b0ec8993e695","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6826","nonce":2915608972},{"type":"v1","addr":"172.21.15.45:6827","nonce":2915608972}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6828","nonce":2915608972},{"type":"v1","addr":"172.21.15.45:6829","nonce":2915608972}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6832","nonce":2915608972},{"type":"v1","addr":"172.21.15.45:6833","nonce":2915608972}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.45:6830","nonce":2915608972},{"type":"v1","addr":"172.21.15.45:6831","nonce":2915608972}]},"public_addr":"172.21.15.45:6827/2915608972","cluster_addr":"172.21.15.45:6829/2915608972","heartbeat_back_addr":"172.21.15.45:6833/2915608972","heartbeat_front_addr":"172.21.15.45:6831/2915608972","state":["exists","up"]},{"osd":4,"uuid":"e538740a-c502-4d61-83b2-1847a9083ff5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6800","nonce":2995343118},{"type":"v1","addr":"172.21.15.165:6801","nonce":2995343118}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6802","nonce":2995343118},{"type":"v1","addr":"172.21.15.165:6803","nonce":2995343118}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6806","nonce":2995343118},{"type":"v1","addr":"172.21.15.165:6807","nonce":2995343118}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6804","nonce":2995343118},{"type":"v1","addr":"172.21.15.165:6805","nonce":2995343118}]},"public_addr":"172.21.15.165:6801/2995343118","cluster_addr":"172.21.15.165:6803/2995343118","heartbeat_back_addr":"172.21.15.165:6807/2995343118","heartbeat_front_addr":"172.21.15.165:6805/2995343118","state":["exists","up"]},{"osd":5,"uuid":"20205cf8-a248-4b9d-9e97-c43f8ded5924","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6808","nonce":2948491063},{"type":"v1","addr":"172.21.15.165:6809","nonce":2948491063}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6810","nonce":2948491063},{"type":"v1","addr":"172.21.15.165:6811","nonce":2948491063}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6814","nonce":2948491063},{"type":"v1","addr":"172.21.15.165:6815","nonce":2948491063}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6812","nonce":2948491063},{"type":"v1","addr":"172.21.15.165:6813","nonce":2948491063}]},"public_addr":"172.21.15.165:6809/2948491063","cluster_addr":"172.21.15.165:6811/2948491063","heartbeat_back_addr":"172.21.15.165:6815/2948491063","heartbeat_front_addr":"172.21.15.165:6813/2948491063","state":["exists","up"]},{"osd":6,"uuid":"7ba074c1-5f25-4279-bee7-5a8cccf7df4b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6816","nonce":1036317166},{"type":"v1","addr":"172.21.15.165:6817","nonce":1036317166}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6818","nonce":1036317166},{"type":"v1","addr":"172.21.15.165:6819","nonce":1036317166}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6822","nonce":1036317166},{"type":"v1","addr":"172.21.15.165:6823","nonce":1036317166}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6820","nonce":1036317166},{"type":"v1","addr":"172.21.15.165:6821","nonce":1036317166}]},"public_addr":"172.21.15.165:6817/1036317166","cluster_addr":"172.21.15.165:6819/1036317166","heartbeat_back_addr":"172.21.15.165:6823/1036317166","heartbeat_front_addr":"172.21.15.165:6821/1036317166","state":["exists","up"]},{"osd":7,"uuid":"9be76121-235e-46bc-8a13-44e2a409b03b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6824","nonce":446054329},{"type":"v1","addr":"172.21.15.165:6825","nonce":446054329}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6826","nonce":446054329},{"type":"v1","addr":"172.21.15.165:6827","nonce":446054329}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6830","nonce":446054329},{"type":"v1","addr":"172.21.15.165:6831","nonce":446054329}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.165:6828","nonce":446054329},{"type":"v1","addr":"172.21.15.165:6829","nonce":446054329}]},"public_addr":"172.21.15.165:6825/446054329","cluster_addr":"172.21.15.165:6827/446054329","heartbeat_back_addr":"172.21.15.165:6831/446054329","heartbeat_front_addr":"172.21.15.165:6829/446054329","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:50:59.927811+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:51:26.369952+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:51:53.856372+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:52:23.587819+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:52:46.069345+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:53:10.166892+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:53:33.859394+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T11:53:58.133456+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.45:0/853583807":"2024-09-18T11:49:50.540848+0000","172.21.15.45:6801/2165128626":"2024-09-18T11:49:02.452594+0000","172.21.15.45:0/3179235283":"2024-09-18T11:49:50.540848+0000","172.21.15.45:0/1137733817":"2024-09-18T11:49:50.540848+0000","172.21.15.45:0/399753365":"2024-09-18T11:49:02.452594+0000","172.21.15.45:0/469356163":"2024-09-18T11:49:02.452594+0000","172.21.15.45:0/2062074897":"2024-09-18T11:49:02.452594+0000","172.21.15.45:6801/881200176":"2024-09-18T11:49:50.540848+0000","172.21.15.45:0/1225275789":"2024-09-18T11:48:32.377129+0000","172.21.15.45:6800/436731029":"2024-09-18T11:48:32.377129+0000","172.21.15.45:6801/436731029":"2024-09-18T11:48:32.377129+0000","172.21.15.45:6800/881200176":"2024-09-18T11:49:50.540848+0000","172.21.15.45:0/2821439202":"2024-09-18T11:48:32.377129+0000","172.21.15.45:6800/2165128626":"2024-09-18T11:49:02.452594+0000","172.21.15.45:0/515207401":"2024-09-18T11:48:32.377129+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T11:54:21.191 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:21 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/2662365123' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T11:54:21.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:21 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/2662365123' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T11:54:21.614 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-17T11:54:21.614 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-17T11:54:21.615 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-17T11:54:21.615 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-17T11:54:21.616 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-17T11:54:21.616 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-17T11:54:21.617 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-17T11:54:21.617 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-17T11:54:22.027 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:54:22.027 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:54:22.027 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:54:22.051 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:54:22.052 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:54:22.060 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:54:22.061 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:54:22.067 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:54:22.446 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:22 smithi045 ceph-mon[23737]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:22.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:22 smithi165 ceph-mon[25702]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:24.347 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:24 smithi045 ceph-mon[23737]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:24.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:24 smithi165 ceph-mon[25702]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:26.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:26 smithi045 ceph-mon[23737]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:26.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:26 smithi165 ceph-mon[25702]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:28.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:28 smithi045 ceph-mon[23737]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:28.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:28 smithi165 ceph-mon[25702]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:30.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:30 smithi165 ceph-mon[25702]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:30.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:30 smithi045 ceph-mon[23737]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:32.181 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:32 smithi045 ceph-mon[23737]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:32.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:32 smithi165 ceph-mon[25702]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:33.233 INFO:teuthology.orchestra.run.smithi045.stdout:236223201288 2024-09-17T11:54:33.234 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-17T11:54:33.993 INFO:teuthology.orchestra.run.smithi045.stdout:81604378658 2024-09-17T11:54:33.993 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-17T11:54:34.323 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:34 smithi045 ceph-mon[23737]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:34.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:34 smithi165 ceph-mon[25702]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:36.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:36 smithi045 ceph-mon[23737]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:36.460 INFO:teuthology.orchestra.run.smithi045.stdout:167503724564 2024-09-17T11:54:36.460 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-17T11:54:36.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:36 smithi165 ceph-mon[25702]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:37.351 INFO:teuthology.orchestra.run.smithi045.stdout:201863462926 2024-09-17T11:54:37.352 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-17T11:54:37.937 INFO:teuthology.orchestra.run.smithi045.stdout:133143986201 2024-09-17T11:54:37.938 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-17T11:54:38.207 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:38 smithi045 ceph-mon[23737]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:38.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:38 smithi165 ceph-mon[25702]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:38.601 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:54:39.266 INFO:teuthology.orchestra.run.smithi045.stdout:38654705709 2024-09-17T11:54:39.267 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-17T11:54:39.579 INFO:teuthology.orchestra.run.smithi045.stdout:111669149725 2024-09-17T11:54:39.580 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-17T11:54:40.344 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:54:40.344 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:54:40.345 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:54:40.345 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:54:40.464 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:40 smithi045 ceph-mon[23737]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:40.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:40 smithi165 ceph-mon[25702]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:40.789 INFO:teuthology.orchestra.run.smithi045.stdout:60129542185 2024-09-17T11:54:40.790 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-17T11:54:42.233 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:42 smithi045 ceph-mon[23737]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:42.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:42 smithi165 ceph-mon[25702]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:43.162 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:54:43.162 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:54:44.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:44 smithi045 ceph-mon[23737]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:44.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:44 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/892294093' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-17T11:54:44.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:44 smithi165 ceph-mon[25702]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:44.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:44 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/892294093' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-17T11:54:44.742 INFO:teuthology.orchestra.run.smithi045.stdout:236223201290 2024-09-17T11:54:44.967 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:54:46.233 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201288 got 236223201290 for osd.7 2024-09-17T11:54:46.234 DEBUG:teuthology.parallel:result is None 2024-09-17T11:54:46.412 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:46 smithi045 ceph-mon[23737]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:46.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:46 smithi165 ceph-mon[25702]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:46.639 INFO:teuthology.orchestra.run.smithi045.stdout:201863462928 2024-09-17T11:54:47.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:47 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/2746695668' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-17T11:54:47.448 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462926 got 201863462928 for osd.6 2024-09-17T11:54:47.448 DEBUG:teuthology.parallel:result is None 2024-09-17T11:54:47.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:47 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/2746695668' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-17T11:54:48.240 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:48 smithi045 ceph-mon[23737]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:48.240 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:48 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/3174114520' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-17T11:54:48.338 INFO:teuthology.orchestra.run.smithi045.stdout:133143986203 2024-09-17T11:54:48.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:48 smithi165 ceph-mon[25702]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:48.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:48 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/3174114520' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-17T11:54:48.944 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986201 got 133143986203 for osd.4 2024-09-17T11:54:48.944 DEBUG:teuthology.parallel:result is None 2024-09-17T11:54:50.429 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:50 smithi045 ceph-mon[23737]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:50.429 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:50 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/1763930747' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-17T11:54:50.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:50 smithi165 ceph-mon[25702]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:50.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:50 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/1763930747' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-17T11:54:51.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:54:51.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:54:51.451 INFO:teuthology.orchestra.run.smithi045.stdout:81604378662 2024-09-17T11:54:51.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:51 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:54:51.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:51 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:54:52.158 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378658 got 81604378662 for osd.2 2024-09-17T11:54:52.158 DEBUG:teuthology.parallel:result is None 2024-09-17T11:54:52.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:52 smithi165 ceph-mon[25702]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:52.545 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:52 smithi045 ceph-mon[23737]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:53.670 INFO:teuthology.orchestra.run.smithi045.stdout:167503724567 2024-09-17T11:54:54.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:54 smithi045 ceph-mon[23737]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:54.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:54 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/2094521520' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-17T11:54:54.488 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724564 got 167503724567 for osd.5 2024-09-17T11:54:54.489 DEBUG:teuthology.parallel:result is None 2024-09-17T11:54:54.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:54 smithi165 ceph-mon[25702]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:54.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:54 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/2094521520' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-17T11:54:55.230 INFO:teuthology.orchestra.run.smithi045.stdout:38654705713 2024-09-17T11:54:55.981 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705709 got 38654705713 for osd.0 2024-09-17T11:54:55.981 DEBUG:teuthology.parallel:result is None 2024-09-17T11:54:56.095 INFO:teuthology.orchestra.run.smithi045.stdout:60129542189 2024-09-17T11:54:56.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:56 smithi045 ceph-mon[23737]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:56.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:56 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/2520599205' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-17T11:54:56.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:56 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/656961365' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-17T11:54:56.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:56 smithi165 ceph-mon[25702]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:56.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:56 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/2520599205' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-17T11:54:56.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:56 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/656961365' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-17T11:54:56.751 INFO:teuthology.orchestra.run.smithi045.stdout:111669149728 2024-09-17T11:54:56.923 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542185 got 60129542189 for osd.1 2024-09-17T11:54:56.923 DEBUG:teuthology.parallel:result is None 2024-09-17T11:54:57.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:57 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/3710898246' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-17T11:54:57.520 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:57 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/3710898246' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-17T11:54:57.548 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149725 got 111669149728 for osd.3 2024-09-17T11:54:57.549 DEBUG:teuthology.parallel:result is None 2024-09-17T11:54:57.549 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-17T11:54:57.549 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-17T11:54:57.771 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:54:58.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:54:58 smithi165 ceph-mon[25702]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:58.571 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:54:58 smithi045 ceph-mon[23737]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:54:59.594 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:54:59.595 INFO:teuthology.orchestra.run.smithi045.stderr:dumped all 2024-09-17T11:55:00.197 INFO:teuthology.orchestra.run.smithi045.stdout:{"pg_ready":true,"pg_map":{"version":201,"stamp":"2024-09-17T11:54:58.645966+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48912,"kb_used_data":6288,"kb_used_omap":0,"kb_used_meta":42560,"kb_avail":749912304,"statfs":{"total":767960285184,"available":767910199296,"internally_reserved":0,"allocated":6438912,"data_stored":3545312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43581440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001880"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-17T11:54:03.715479+0000","last_change":"2024-09-17T11:54:03.715479+0000","last_active":"2024-09-17T11:54:03.715479+0000","last_peered":"2024-09-17T11:54:03.715479+0000","last_clean":"2024-09-17T11:54:03.715479+0000","last_became_active":"2024-09-17T11:54:03.714972+0000","last_became_peered":"2024-09-17T11:54:03.714972+0000","last_unstale":"2024-09-17T11:54:03.715479+0000","last_undegraded":"2024-09-17T11:54:03.715479+0000","last_fullsized":"2024-09-17T11:54:03.715479+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T11:51:57.108156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T11:51:57.108156+0000","last_clean_scrub_stamp":"2024-09-17T11:51:57.108156+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-18T22:07:59.449671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201293,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46700000000000003}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.505}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34899999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41399999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58399999999999996}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42899999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48099999999999998}]}]},{"osd":6,"up_from":47,"seq":201863462931,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6172,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738980,"statfs":{"total":95995035648,"available":95988715520,"internally_reserved":0,"allocated":1134592,"data_stored":773573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 11:54:38 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.61599999999999999}]},{"osd":1,"last update":"Tue Sep 17 11:54:38 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.54000000000000004}]},{"osd":2,"last update":"Tue Sep 17 11:54:38 2024","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.212,"15min":1.212},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":15.999000000000001,"5min":15.999000000000001,"15min":15.999000000000001},"last":0.628},{"interface":"front","average":{"1min":1.234,"5min":1.234,"15min":1.234},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":15.987,"5min":15.987,"15min":15.987},"last":0.53200000000000003}]},{"osd":3,"last update":"Tue Sep 17 11:54:38 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.58899999999999997}]},{"osd":4,"last update":"Tue Sep 17 11:54:38 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.433}]},{"osd":5,"last update":"Tue Sep 17 11:54:38 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.38400000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47699999999999998}]}]},{"osd":1,"up_from":14,"seq":60129542189,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6360,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738792,"statfs":{"total":95995035648,"available":95988523008,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 11:54:30 2024","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.34399999999999997,"15min":0.33000000000000002},"min":{"1min":0.24299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.376,"15min":0.35199999999999998},"min":{"1min":0.30099999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.44500000000000001}]},{"osd":2,"last update":"Tue Sep 17 11:54:02 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.39400000000000002,"15min":0.38700000000000001},"min":{"1min":0.28000000000000003,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.377},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.40899999999999997,"15min":0.40100000000000002},"min":{"1min":0.312,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.52200000000000002}]},{"osd":3,"last update":"Tue Sep 17 11:54:30 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.435,"15min":0.44},"min":{"1min":0.32500000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.53000000000000003,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.497},{"interface":"front","average":{"1min":0.44,"5min":0.44500000000000001,"15min":0.44600000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.42999999999999999}]},{"osd":4,"last update":"Tue Sep 17 11:54:50 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.59399999999999997,"15min":0.59899999999999998},"min":{"1min":0.44400000000000001,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.82099999999999995,"5min":0.875,"15min":0.875},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.59799999999999998,"15min":0.59999999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.67700000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.55200000000000005}]},{"osd":5,"last update":"Tue Sep 17 11:54:14 2024","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.57999999999999996}]},{"osd":6,"last update":"Tue Sep 17 11:54:39 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.61499999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64000000000000001}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Sep 17 11:54:36 2024","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.34000000000000002,"15min":0.32300000000000001},"min":{"1min":0.27100000000000002,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.45600000000000002,"5min":0.628,"15min":0.628},"last":0.312},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.35599999999999998,"15min":0.33400000000000002},"min":{"1min":0.26700000000000002,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.28199999999999997}]},{"osd":2,"last update":"Tue Sep 17 11:53:59 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.376,"15min":0.36599999999999999},"min":{"1min":0.33900000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.41099999999999998,"15min":0.40300000000000002},"min":{"1min":0.33400000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.27000000000000002}]},{"osd":3,"last update":"Tue Sep 17 11:54:30 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.441,"15min":0.44500000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.50900000000000001,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.307},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.48499999999999999,"15min":0.495},"min":{"1min":0.34599999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.57799999999999996,"5min":0.751,"15min":0.751},"last":0.32400000000000001}]},{"osd":4,"last update":"Tue Sep 17 11:54:47 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.58499999999999996,"15min":0.57699999999999996},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.75900000000000001,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.60599999999999998,"15min":0.60399999999999998},"min":{"1min":0.48099999999999998,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.80500000000000005,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.57399999999999995}]},{"osd":5,"last update":"Tue Sep 17 11:54:16 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.59599999999999997}]},{"osd":6,"last update":"Tue Sep 17 11:54:36 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.622}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58399999999999996}]}]},{"osd":2,"up_from":19,"seq":81604378664,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6292,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738860,"statfs":{"total":95995035648,"available":95988592640,"internally_reserved":0,"allocated":602112,"data_stored":244249,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 11:53:59 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.34799999999999998,"15min":0.33300000000000002},"min":{"1min":0.309,"5min":0.22,"15min":0.22},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.36499999999999999,"15min":0.34899999999999998},"min":{"1min":0.30299999999999999,"5min":0.251,"15min":0.251},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.311}]},{"osd":1,"last update":"Tue Sep 17 11:53:59 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.35599999999999998,"15min":0.34599999999999997},"min":{"1min":0.32300000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.441},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.41199999999999998,"15min":0.40100000000000002},"min":{"1min":0.38700000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.374}]},{"osd":3,"last update":"Tue Sep 17 11:54:30 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.48399999999999999,"15min":0.47999999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.502,"5min":0.45800000000000002,"15min":0.45100000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.2629999999999999,"5min":1.2629999999999999,"15min":1.2629999999999999},"last":0.42599999999999999}]},{"osd":4,"last update":"Tue Sep 17 11:54:51 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.44900000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.226,"5min":1.228,"15min":1.228},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.67300000000000004,"15min":0.67500000000000004},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.0800000000000001,"5min":1.127,"15min":1.127},"last":0.71999999999999997}]},{"osd":5,"last update":"Tue Sep 17 11:54:15 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":1.0680000000000001,"5min":1.0680000000000001,"15min":1.0680000000000001},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":1.1819999999999999,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":0.56599999999999995}]},{"osd":6,"last update":"Tue Sep 17 11:54:36 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.034,"5min":1.034,"15min":1.034},"last":0.70299999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61899999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149729,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 11:54:26 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.39500000000000002,"15min":0.39100000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.41499999999999998,"15min":0.40999999999999998},"min":{"1min":0.28699999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.32100000000000001}]},{"osd":1,"last update":"Tue Sep 17 11:54:26 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.42999999999999999,"15min":0.42799999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.436,"5min":0.42799999999999999,"15min":0.42599999999999999},"min":{"1min":0.28899999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.66000000000000003,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.38800000000000001}]},{"osd":2,"last update":"Tue Sep 17 11:54:26 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.47399999999999998,"15min":0.47799999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.433},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.438,"15min":0.434},"min":{"1min":0.31,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.40500000000000003}]},{"osd":4,"last update":"Tue Sep 17 11:54:51 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.59799999999999998,"15min":0.59699999999999998},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.621,"5min":0.61499999999999999,"15min":0.61399999999999999},"min":{"1min":0.505,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.55300000000000005}]},{"osd":5,"last update":"Tue Sep 17 11:54:14 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.50600000000000001}]},{"osd":6,"last update":"Tue Sep 17 11:54:38 2024","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.499},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.60299999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56699999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986206,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5784,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739368,"statfs":{"total":95995035648,"available":95989112832,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 11:54:50 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.59899999999999998,"15min":0.60299999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.78900000000000003,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.59799999999999998,"15min":0.59999999999999998},"min":{"1min":0.439,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.71099999999999997,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.63400000000000001}]},{"osd":1,"last update":"Tue Sep 17 11:54:50 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.57999999999999996,"15min":0.58399999999999996},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.749,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.61499999999999999,"15min":0.61699999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.73699999999999999,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.55300000000000005}]},{"osd":2,"last update":"Tue Sep 17 11:54:50 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.621,"15min":0.623},"min":{"1min":0.376,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.77100000000000002,"5min":0.999,"15min":0.999},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.60299999999999998,"15min":0.60599999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.75,"5min":1.048,"15min":1.048},"last":0.58299999999999996}]},{"osd":3,"last update":"Tue Sep 17 11:54:50 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.60899999999999999,"15min":0.61499999999999999},"min":{"1min":0.41099999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.79800000000000004,"5min":1.097,"15min":1.097},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.59899999999999998,"15min":0.59799999999999998},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.78400000000000003,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.70199999999999996}]},{"osd":5,"last update":"Tue Sep 17 11:54:17 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.64900000000000002}]},{"osd":6,"last update":"Tue Sep 17 11:54:41 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.66500000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68999999999999995}]}]},{"osd":5,"up_from":39,"seq":167503724569,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5652,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739500,"statfs":{"total":95995035648,"available":95989248000,"internally_reserved":0,"allocated":602112,"data_stored":244587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 11:54:13 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.51900000000000002}]},{"osd":1,"last update":"Tue Sep 17 11:54:13 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.56699999999999995}]},{"osd":2,"last update":"Tue Sep 17 11:54:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.57199999999999995}]},{"osd":3,"last update":"Tue Sep 17 11:54:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.54400000000000004}]},{"osd":4,"last update":"Tue Sep 17 11:54:13 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.376}]},{"osd":6,"last update":"Tue Sep 17 11:54:36 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.46800000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.432}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-17T11:55:00.199 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-17T11:55:00.416 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:55:00.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:00 smithi165 ceph-mon[25702]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:00.537 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:00 smithi045 ceph-mon[23737]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:01.470 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:01 smithi045 ceph-mon[23737]: from='client.14450 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T11:55:01.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:01 smithi165 ceph-mon[25702]: from='client.14450 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T11:55:02.272 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:55:02.274 INFO:teuthology.orchestra.run.smithi045.stderr:dumped all 2024-09-17T11:55:02.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:02 smithi165 ceph-mon[25702]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:02.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:02 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:55:02.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:02 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:02.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:02 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:55:02.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:02 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:02.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:02 smithi045 ceph-mon[23737]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:02.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:02 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:55:02.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:02 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:02.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:02 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:55:02.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:02 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:02.911 INFO:teuthology.orchestra.run.smithi045.stdout:{"pg_ready":true,"pg_map":{"version":202,"stamp":"2024-09-17T11:55:00.646335+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48912,"kb_used_data":6288,"kb_used_omap":0,"kb_used_meta":42560,"kb_avail":749912304,"statfs":{"total":767960285184,"available":767910199296,"internally_reserved":0,"allocated":6438912,"data_stored":3545312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43581440},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001939"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-17T11:54:03.715479+0000","last_change":"2024-09-17T11:54:03.715479+0000","last_active":"2024-09-17T11:54:03.715479+0000","last_peered":"2024-09-17T11:54:03.715479+0000","last_clean":"2024-09-17T11:54:03.715479+0000","last_became_active":"2024-09-17T11:54:03.714972+0000","last_became_peered":"2024-09-17T11:54:03.714972+0000","last_unstale":"2024-09-17T11:54:03.715479+0000","last_undegraded":"2024-09-17T11:54:03.715479+0000","last_fullsized":"2024-09-17T11:54:03.715479+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T11:51:57.108156+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T11:51:57.108156+0000","last_clean_scrub_stamp":"2024-09-17T11:51:57.108156+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-18T22:07:59.449671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201294,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47099999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56599999999999995}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52300000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.443}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54700000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63900000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68000000000000005}]}]},{"osd":6,"up_from":47,"seq":201863462931,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6172,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738980,"statfs":{"total":95995035648,"available":95988715520,"internally_reserved":0,"allocated":1134592,"data_stored":773573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 11:54:38 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.61599999999999999}]},{"osd":1,"last update":"Tue Sep 17 11:54:38 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.54000000000000004}]},{"osd":2,"last update":"Tue Sep 17 11:54:38 2024","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.212,"15min":1.212},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":15.999000000000001,"5min":15.999000000000001,"15min":15.999000000000001},"last":0.628},{"interface":"front","average":{"1min":1.234,"5min":1.234,"15min":1.234},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":15.987,"5min":15.987,"15min":15.987},"last":0.53200000000000003}]},{"osd":3,"last update":"Tue Sep 17 11:54:38 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.58899999999999997}]},{"osd":4,"last update":"Tue Sep 17 11:54:38 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.433}]},{"osd":5,"last update":"Tue Sep 17 11:54:38 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.38400000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47699999999999998}]}]},{"osd":1,"up_from":14,"seq":60129542190,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6360,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738792,"statfs":{"total":95995035648,"available":95988523008,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 11:54:30 2024","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.34399999999999997,"15min":0.33000000000000002},"min":{"1min":0.24299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.376,"15min":0.35199999999999998},"min":{"1min":0.30099999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.29999999999999999}]},{"osd":2,"last update":"Tue Sep 17 11:54:02 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.39400000000000002,"15min":0.38700000000000001},"min":{"1min":0.28000000000000003,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.40899999999999997,"15min":0.40100000000000002},"min":{"1min":0.312,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.38200000000000001}]},{"osd":3,"last update":"Tue Sep 17 11:54:30 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.435,"15min":0.44},"min":{"1min":0.32500000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.53000000000000003,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.44,"5min":0.44500000000000001,"15min":0.44600000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.32900000000000001}]},{"osd":4,"last update":"Tue Sep 17 11:54:50 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.59399999999999997,"15min":0.59899999999999998},"min":{"1min":0.44400000000000001,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.82099999999999995,"5min":0.875,"15min":0.875},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.59799999999999998,"15min":0.59999999999999998},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.67700000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.44400000000000001}]},{"osd":5,"last update":"Tue Sep 17 11:54:14 2024","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.51600000000000001}]},{"osd":6,"last update":"Tue Sep 17 11:54:39 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.59999999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58999999999999997}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Sep 17 11:54:36 2024","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.34000000000000002,"15min":0.32300000000000001},"min":{"1min":0.27100000000000002,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":0.45600000000000002,"5min":0.628,"15min":0.628},"last":0.312},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.35599999999999998,"15min":0.33400000000000002},"min":{"1min":0.26700000000000002,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.28199999999999997}]},{"osd":2,"last update":"Tue Sep 17 11:53:59 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.376,"15min":0.36599999999999999},"min":{"1min":0.33900000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.41099999999999998,"15min":0.40300000000000002},"min":{"1min":0.33400000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.27000000000000002}]},{"osd":3,"last update":"Tue Sep 17 11:54:30 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.441,"15min":0.44500000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.50900000000000001,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.307},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.48499999999999999,"15min":0.495},"min":{"1min":0.34599999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.57799999999999996,"5min":0.751,"15min":0.751},"last":0.32400000000000001}]},{"osd":4,"last update":"Tue Sep 17 11:54:47 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.58499999999999996,"15min":0.57699999999999996},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.75900000000000001,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.60599999999999998,"15min":0.60399999999999998},"min":{"1min":0.48099999999999998,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":0.80500000000000005,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.57399999999999995}]},{"osd":5,"last update":"Tue Sep 17 11:54:16 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.59599999999999997}]},{"osd":6,"last update":"Tue Sep 17 11:54:36 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.622}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58399999999999996}]}]},{"osd":2,"up_from":19,"seq":81604378664,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6292,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738860,"statfs":{"total":95995035648,"available":95988592640,"internally_reserved":0,"allocated":602112,"data_stored":244249,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 11:53:59 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.34799999999999998,"15min":0.33300000000000002},"min":{"1min":0.309,"5min":0.22,"15min":0.22},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.36499999999999999,"15min":0.34899999999999998},"min":{"1min":0.30299999999999999,"5min":0.251,"15min":0.251},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.311}]},{"osd":1,"last update":"Tue Sep 17 11:53:59 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.35599999999999998,"15min":0.34599999999999997},"min":{"1min":0.32300000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"last":0.441},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.41199999999999998,"15min":0.40100000000000002},"min":{"1min":0.38700000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.374}]},{"osd":3,"last update":"Tue Sep 17 11:54:30 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.48399999999999999,"15min":0.47999999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.502,"5min":0.45800000000000002,"15min":0.45100000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.2629999999999999,"5min":1.2629999999999999,"15min":1.2629999999999999},"last":0.42599999999999999}]},{"osd":4,"last update":"Tue Sep 17 11:54:51 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.44900000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.226,"5min":1.228,"15min":1.228},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.67300000000000004,"15min":0.67500000000000004},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":1.0800000000000001,"5min":1.127,"15min":1.127},"last":0.71999999999999997}]},{"osd":5,"last update":"Tue Sep 17 11:54:15 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":1.0680000000000001,"5min":1.0680000000000001,"15min":1.0680000000000001},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.505,"5min":0.505,"15min":0.505},"max":{"1min":1.1819999999999999,"5min":1.1819999999999999,"15min":1.1819999999999999},"last":0.56599999999999995}]},{"osd":6,"last update":"Tue Sep 17 11:54:36 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.034,"5min":1.034,"15min":1.034},"last":0.70299999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61899999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149729,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 11:54:26 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.39500000000000002,"15min":0.39100000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.41499999999999998,"15min":0.40999999999999998},"min":{"1min":0.28699999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"last":0.32100000000000001}]},{"osd":1,"last update":"Tue Sep 17 11:54:26 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.42999999999999999,"15min":0.42799999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.436,"5min":0.42799999999999999,"15min":0.42599999999999999},"min":{"1min":0.28899999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.66000000000000003,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.38800000000000001}]},{"osd":2,"last update":"Tue Sep 17 11:54:26 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.47399999999999998,"15min":0.47799999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.433},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.438,"15min":0.434},"min":{"1min":0.31,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.40500000000000003}]},{"osd":4,"last update":"Tue Sep 17 11:54:51 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.59799999999999998,"15min":0.59699999999999998},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.621,"5min":0.61499999999999999,"15min":0.61399999999999999},"min":{"1min":0.505,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.55300000000000005}]},{"osd":5,"last update":"Tue Sep 17 11:54:14 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.50600000000000001}]},{"osd":6,"last update":"Tue Sep 17 11:54:38 2024","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.499},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.60299999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56699999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986206,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5784,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739368,"statfs":{"total":95995035648,"available":95989112832,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 11:54:50 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.59899999999999998,"15min":0.60299999999999998},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.78900000000000003,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.59799999999999998,"15min":0.59999999999999998},"min":{"1min":0.439,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.71099999999999997,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.63400000000000001}]},{"osd":1,"last update":"Tue Sep 17 11:54:50 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.57999999999999996,"15min":0.58399999999999996},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.749,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.61499999999999999,"15min":0.61699999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.73699999999999999,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.55300000000000005}]},{"osd":2,"last update":"Tue Sep 17 11:54:50 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.621,"15min":0.623},"min":{"1min":0.376,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.77100000000000002,"5min":0.999,"15min":0.999},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.60299999999999998,"15min":0.60599999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.75,"5min":1.048,"15min":1.048},"last":0.58299999999999996}]},{"osd":3,"last update":"Tue Sep 17 11:54:50 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.60899999999999999,"15min":0.61499999999999999},"min":{"1min":0.41099999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.79800000000000004,"5min":1.097,"15min":1.097},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.59899999999999998,"15min":0.59799999999999998},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.78400000000000003,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.70199999999999996}]},{"osd":5,"last update":"Tue Sep 17 11:54:17 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.64900000000000002}]},{"osd":6,"last update":"Tue Sep 17 11:54:41 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.66500000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68999999999999995}]}]},{"osd":5,"up_from":39,"seq":167503724569,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5652,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739500,"statfs":{"total":95995035648,"available":95989248000,"internally_reserved":0,"allocated":602112,"data_stored":244587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 11:54:13 2024","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.51900000000000002}]},{"osd":1,"last update":"Tue Sep 17 11:54:13 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.56699999999999995}]},{"osd":2,"last update":"Tue Sep 17 11:54:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.57199999999999995}]},{"osd":3,"last update":"Tue Sep 17 11:54:13 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.54400000000000004}]},{"osd":4,"last update":"Tue Sep 17 11:54:13 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.376}]},{"osd":6,"last update":"Tue Sep 17 11:54:36 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.46800000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.432}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-17T11:55:02.913 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-17T11:55:02.913 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-17T11:55:02.913 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-17T11:55:02.914 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-17T11:55:03.132 INFO:teuthology.orchestra.run.smithi045.stderr:Inferring config /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/mon.smithi045/config 2024-09-17T11:55:03.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:03 smithi165 ceph-mon[25702]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T11:55:03.513 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:03 smithi045 ceph-mon[23737]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T11:55:04.507 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:04 smithi045 ceph-mon[23737]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:04.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:04 smithi165 ceph-mon[25702]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:04.979 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:55:04.979 INFO:teuthology.orchestra.run.smithi045.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-17T11:55:05.280 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:05 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/3006443506' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-17T11:55:05.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:05 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/3006443506' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-17T11:55:05.580 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-17T11:55:05.580 INFO:tasks.cephadm:Setup complete, yielding 2024-09-17T11:55:05.581 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T11:55:05.591 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi045.front.sepia.ceph.com 2024-09-17T11:55:05.591 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- bash -c 'ceph orch status' 2024-09-17T11:55:06.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:06 smithi165 ceph-mon[25702]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:06.538 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:06 smithi045 ceph-mon[23737]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:07.564 INFO:teuthology.orchestra.run.smithi045.stdout:Backend: cephadm 2024-09-17T11:55:07.565 INFO:teuthology.orchestra.run.smithi045.stdout:Available: Yes 2024-09-17T11:55:07.565 INFO:teuthology.orchestra.run.smithi045.stdout:Paused: No 2024-09-17T11:55:08.177 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ps' 2024-09-17T11:55:08.483 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:08 smithi045 ceph-mon[23737]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:08.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:08 smithi165 ceph-mon[25702]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:09.417 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:09 smithi045 ceph-mon[23737]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:09.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:09 smithi165 ceph-mon[25702]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:10.141 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T11:55:10.141 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (4m) 2m ago 5m 15.4M - ba2b418f427c c97407c70fb6 2024-09-17T11:55:10.141 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (5m) 2m ago 5m 6413k - 17.2.0 e1d6a67b021e 8c510943b55e 2024-09-17T11:55:10.142 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (5m) 71s ago 5m 6409k - 17.2.0 e1d6a67b021e 687c24a87b70 2024-09-17T11:55:10.142 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (4m) 2m ago 5m 44.8M - 8.3.5 dad864ee21e9 7d87d4101e56 2024-09-17T11:55:10.142 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:9283 running (7m) 2m ago 7m 448M - 17.2.0 e1d6a67b021e 5f496de39b08 2024-09-17T11:55:10.142 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283 running (5m) 71s ago 5m 410M - 17.2.0 e1d6a67b021e fefbdcc12a47 2024-09-17T11:55:10.142 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (7m) 2m ago 7m 43.0M 2048M 17.2.0 e1d6a67b021e c4256bdf51b7 2024-09-17T11:55:10.142 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (5m) 71s ago 5m 39.2M 2048M 17.2.0 e1d6a67b021e 9219263d3aa8 2024-09-17T11:55:10.142 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (5m) 2m ago 5m 17.6M - 1dbe0e931976 09dea232cb6b 2024-09-17T11:55:10.142 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (4m) 71s ago 4m 16.7M - 1dbe0e931976 fdb9449adb13 2024-09-17T11:55:10.142 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (4m) 2m ago 4m 33.4M 3217M 17.2.0 e1d6a67b021e e0ce4d29dc64 2024-09-17T11:55:10.142 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (3m) 2m ago 3m 34.7M 3217M 17.2.0 e1d6a67b021e da8bda7b47b3 2024-09-17T11:55:10.142 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (3m) 2m ago 3m 31.7M 3217M 17.2.0 e1d6a67b021e 21fe0d9e5ced 2024-09-17T11:55:10.143 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (2m) 2m ago 2m 30.2M 3217M 17.2.0 e1d6a67b021e 92a2389488e4 2024-09-17T11:55:10.143 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (2m) 71s ago 2m 35.7M 3984M 17.2.0 e1d6a67b021e e29eb74df25a 2024-09-17T11:55:10.143 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (2m) 71s ago 2m 33.0M 3984M 17.2.0 e1d6a67b021e 5d900eb9165b 2024-09-17T11:55:10.143 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (100s) 71s ago 100s 33.1M 3984M 17.2.0 e1d6a67b021e fae3e1bae466 2024-09-17T11:55:10.143 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (76s) 71s ago 76s 10.3M 3984M 17.2.0 e1d6a67b021e 71b9f2cdb810 2024-09-17T11:55:10.143 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (4m) 2m ago 4m 52.2M - 514e6a882f6e 528c529e18d6 2024-09-17T11:55:10.481 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:10 smithi045 ceph-mon[23737]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:10.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:10 smithi165 ceph-mon[25702]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:10.748 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ls' 2024-09-17T11:55:11.327 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:11 smithi045 ceph-mon[23737]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:11.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:11 smithi165 ceph-mon[25702]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:12.291 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:12 smithi045 ceph-mon[23737]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:12.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:12 smithi165 ceph-mon[25702]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:12.712 INFO:teuthology.orchestra.run.smithi045.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-17T11:55:12.712 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-09-17T11:55:12.712 INFO:teuthology.orchestra.run.smithi045.stdout:crash 2/2 2m ago 6m * 2024-09-17T11:55:12.713 INFO:teuthology.orchestra.run.smithi045.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-09-17T11:55:12.713 INFO:teuthology.orchestra.run.smithi045.stdout:mgr 2/2 2m ago 6m count:2 2024-09-17T11:55:12.713 INFO:teuthology.orchestra.run.smithi045.stdout:mon 2/2 2m ago 5m smithi045:172.21.15.45=smithi045;smithi165:172.21.15.165=smithi165;count:2 2024-09-17T11:55:12.713 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-09-17T11:55:12.713 INFO:teuthology.orchestra.run.smithi045.stdout:osd 8 2m ago - 2024-09-17T11:55:12.713 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-09-17T11:55:13.356 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- bash -c 'ceph orch host ls' 2024-09-17T11:55:14.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:14 smithi045 ceph-mon[23737]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:14.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:14 smithi045 ceph-mon[23737]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:14.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:14 smithi165 ceph-mon[25702]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:14.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:14 smithi165 ceph-mon[25702]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:15.349 INFO:teuthology.orchestra.run.smithi045.stdout:HOST ADDR LABELS STATUS 2024-09-17T11:55:15.349 INFO:teuthology.orchestra.run.smithi045.stdout:smithi045 172.21.15.45 2024-09-17T11:55:15.349 INFO:teuthology.orchestra.run.smithi045.stdout:smithi165 172.21.15.165 2024-09-17T11:55:15.349 INFO:teuthology.orchestra.run.smithi045.stdout:2 hosts in cluster 2024-09-17T11:55:15.925 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- bash -c 'ceph orch device ls' 2024-09-17T11:55:16.486 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:16 smithi045 ceph-mon[23737]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:16.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:16 smithi165 ceph-mon[25702]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:17.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:17 smithi165 ceph-mon[25702]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:17.529 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:17 smithi045 ceph-mon[23737]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:17.983 INFO:teuthology.orchestra.run.smithi045.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-09-17T11:55:17.984 INFO:teuthology.orchestra.run.smithi045.stdout:smithi045 /dev/nvme0n1 ssd INTEL_SSDPED1D480GA_PHMB75120098480DGN 480G LVM detected, locked 2024-09-17T11:55:17.984 INFO:teuthology.orchestra.run.smithi045.stdout:smithi165 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT623300CE400BGN 400G LVM detected, locked 2024-09-17T11:55:18.338 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:18 smithi045 ceph-mon[23737]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:18.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:18 smithi165 ceph-mon[25702]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:18.747 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-17T11:55:18.759 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi045.front.sepia.ceph.com 2024-09-17T11:55:18.760 DEBUG:teuthology.orchestra.run.smithi045:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-17T11:55:18.788 INFO:teuthology.orchestra.run.smithi045.stderr:+ systemctl stop nfs-server 2024-09-17T11:55:18.796 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi165.front.sepia.ceph.com 2024-09-17T11:55:18.796 DEBUG:teuthology.orchestra.run.smithi165:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-17T11:55:18.827 INFO:teuthology.orchestra.run.smithi165.stderr:+ systemctl stop nfs-server 2024-09-17T11:55:18.836 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T11:55:18.844 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi045.front.sepia.ceph.com 2024-09-17T11:55:18.844 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-09-17T11:55:19.409 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:19 smithi045 ceph-mon[23737]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:19.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:19 smithi165 ceph-mon[25702]: from='client.14478 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:20.412 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:20 smithi045 ceph-mon[23737]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:20.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:20 smithi165 ceph-mon[25702]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:22.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:21 smithi045 ceph-mon[23737]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:22.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:21 smithi045 ceph-mon[23737]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:22.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:21 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-17T11:55:22.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:21 smithi165 ceph-mon[25702]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:22.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:21 smithi165 ceph-mon[25702]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:22.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:21 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-17T11:55:22.711 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:22 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045[23733]: 2024-09-17T11:55:22.359+0000 7efe8acfb700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T11:55:23.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:22 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-17T11:55:23.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:22 smithi045 ceph-mon[23737]: osdmap e60: 8 total, 8 up, 8 in 2024-09-17T11:55:23.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:22 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-17T11:55:23.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:22 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-17T11:55:23.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:22 smithi045 ceph-mon[23737]: osdmap e61: 8 total, 8 up, 8 in 2024-09-17T11:55:23.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:22 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-17T11:55:23.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:22 smithi045 ceph-mon[23737]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T11:55:23.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:22 smithi045 ceph-mon[23737]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-17T11:55:23.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:22 smithi045 ceph-mon[23737]: osdmap e62: 8 total, 8 up, 8 in 2024-09-17T11:55:23.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:22 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-17T11:55:23.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:22 smithi045 ceph-mon[23737]: fsmap foofs:0 2024-09-17T11:55:23.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:22 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:23.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:22 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:55:23.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:22 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:23.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:22 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:55:23.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:22 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:23.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:22 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:23.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:22 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi045.olwaas", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T11:55:23.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:22 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi045.olwaas", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-17T11:55:23.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:22 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:23.151 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-09-17T11:55:23.162 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-09-17T11:55:23.162 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-17T11:55:23.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:22 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-17T11:55:23.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:22 smithi165 ceph-mon[25702]: osdmap e60: 8 total, 8 up, 8 in 2024-09-17T11:55:23.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:22 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-17T11:55:23.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:22 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-17T11:55:23.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:22 smithi165 ceph-mon[25702]: osdmap e61: 8 total, 8 up, 8 in 2024-09-17T11:55:23.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:22 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-17T11:55:23.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:22 smithi165 ceph-mon[25702]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T11:55:23.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:22 smithi165 ceph-mon[25702]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-17T11:55:23.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:22 smithi165 ceph-mon[25702]: osdmap e62: 8 total, 8 up, 8 in 2024-09-17T11:55:23.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:22 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-17T11:55:23.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:22 smithi165 ceph-mon[25702]: fsmap foofs:0 2024-09-17T11:55:23.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:22 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:23.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:22 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:55:23.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:22 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:23.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:22 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:55:23.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:22 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:23.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:22 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:23.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:22 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi045.olwaas", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T11:55:23.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:22 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi045.olwaas", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-17T11:55:23.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:22 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:23.974 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:23 smithi045 ceph-mon[23737]: Saving service mds.foofs spec with placement count:2 2024-09-17T11:55:23.974 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:23 smithi045 ceph-mon[23737]: Deploying daemon mds.foofs.smithi045.olwaas on smithi045 2024-09-17T11:55:23.974 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:23 smithi045 ceph-mon[23737]: pgmap v216: 65 pgs: 4 creating+peering, 1 active+clean, 60 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:23.975 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:23 smithi045 ceph-mon[23737]: osdmap e63: 8 total, 8 up, 8 in 2024-09-17T11:55:24.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:23 smithi165 ceph-mon[25702]: Saving service mds.foofs spec with placement count:2 2024-09-17T11:55:24.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:23 smithi165 ceph-mon[25702]: Deploying daemon mds.foofs.smithi045.olwaas on smithi045 2024-09-17T11:55:24.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:23 smithi165 ceph-mon[25702]: pgmap v216: 65 pgs: 4 creating+peering, 1 active+clean, 60 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:24.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:23 smithi165 ceph-mon[25702]: osdmap e63: 8 total, 8 up, 8 in 2024-09-17T11:55:25.558 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:25 smithi045 ceph-mon[23737]: osdmap e64: 8 total, 8 up, 8 in 2024-09-17T11:55:25.558 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:25 smithi045 ceph-mon[23737]: pgmap v219: 65 pgs: 15 creating+peering, 22 active+clean, 28 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:25.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:25 smithi165 ceph-mon[25702]: osdmap e64: 8 total, 8 up, 8 in 2024-09-17T11:55:25.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:25 smithi165 ceph-mon[25702]: pgmap v219: 65 pgs: 15 creating+peering, 22 active+clean, 28 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:26.126 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:55:26.126 INFO:teuthology.orchestra.run.smithi045.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-17T11:48:51.619752Z", "last_refresh": "2024-09-17T11:52:33.768295Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-17T11:48:45.016215Z", "last_refresh": "2024-09-17T11:52:33.768362Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-17T11:48:47.912727Z", "last_refresh": "2024-09-17T11:52:33.768426Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-17T11:55:22.384110Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-17T11:55:22.373606Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-17T11:48:43.055030Z", "last_refresh": "2024-09-17T11:52:33.768221Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi045:172.21.15.45=smithi045", "smithi165:172.21.15.165=smithi165"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-17T11:49:34.840638Z", "last_refresh": "2024-09-17T11:52:33.768076Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-17T11:48:50.361555Z", "last_refresh": "2024-09-17T11:52:33.768489Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-17T11:52:33.768613Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-17T11:48:46.212310Z", "last_refresh": "2024-09-17T11:52:33.768551Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-17T11:55:27.031 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-17T11:55:27.348 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:27 smithi165 ceph-mon[25702]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T11:55:27.348 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:27 smithi165 ceph-mon[25702]: mds.? [v2:172.21.15.45:6834/558522566,v1:172.21.15.45:6835/558522566] up:boot 2024-09-17T11:55:27.348 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:27 smithi165 ceph-mon[25702]: daemon mds.foofs.smithi045.olwaas assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-17T11:55:27.348 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:27 smithi165 ceph-mon[25702]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T11:55:27.348 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:27 smithi165 ceph-mon[25702]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-17T11:55:27.348 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:27 smithi165 ceph-mon[25702]: Cluster is now healthy 2024-09-17T11:55:27.348 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:27 smithi165 ceph-mon[25702]: fsmap foofs:0 1 up:standby 2024-09-17T11:55:27.348 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:27 smithi165 ceph-mon[25702]: fsmap foofs:1 {0=foofs.smithi045.olwaas=up:creating} 2024-09-17T11:55:27.348 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:27 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi045.olwaas"}]: dispatch 2024-09-17T11:55:27.348 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:27 smithi165 ceph-mon[25702]: daemon mds.foofs.smithi045.olwaas is now active in filesystem foofs as rank 0 2024-09-17T11:55:27.349 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:27 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:27.349 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:27 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi165.czpmgd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T11:55:27.349 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:27 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi165.czpmgd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-17T11:55:27.349 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:27 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:27.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:27 smithi045 ceph-mon[23737]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T11:55:27.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:27 smithi045 ceph-mon[23737]: mds.? [v2:172.21.15.45:6834/558522566,v1:172.21.15.45:6835/558522566] up:boot 2024-09-17T11:55:27.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:27 smithi045 ceph-mon[23737]: daemon mds.foofs.smithi045.olwaas assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-17T11:55:27.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:27 smithi045 ceph-mon[23737]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T11:55:27.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:27 smithi045 ceph-mon[23737]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-17T11:55:27.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:27 smithi045 ceph-mon[23737]: Cluster is now healthy 2024-09-17T11:55:27.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:27 smithi045 ceph-mon[23737]: fsmap foofs:0 1 up:standby 2024-09-17T11:55:27.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:27 smithi045 ceph-mon[23737]: fsmap foofs:1 {0=foofs.smithi045.olwaas=up:creating} 2024-09-17T11:55:27.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:27 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi045.olwaas"}]: dispatch 2024-09-17T11:55:27.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:27 smithi045 ceph-mon[23737]: daemon mds.foofs.smithi045.olwaas is now active in filesystem foofs as rank 0 2024-09-17T11:55:27.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:27 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:27.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:27 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi165.czpmgd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T11:55:27.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:27 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi165.czpmgd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-17T11:55:27.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:27 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:28.032 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-17T11:55:28.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:28 smithi045 ceph-mon[23737]: Deploying daemon mds.foofs.smithi165.czpmgd on smithi165 2024-09-17T11:55:28.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:28 smithi045 ceph-mon[23737]: pgmap v220: 65 pgs: 15 creating+peering, 50 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:28.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:28 smithi045 ceph-mon[23737]: mds.? [v2:172.21.15.45:6834/558522566,v1:172.21.15.45:6835/558522566] up:active 2024-09-17T11:55:28.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:28 smithi045 ceph-mon[23737]: fsmap foofs:1 {0=foofs.smithi045.olwaas=up:active} 2024-09-17T11:55:28.448 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:28 smithi165 ceph-mon[25702]: Deploying daemon mds.foofs.smithi165.czpmgd on smithi165 2024-09-17T11:55:28.449 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:28 smithi165 ceph-mon[25702]: pgmap v220: 65 pgs: 15 creating+peering, 50 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:28.449 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:28 smithi165 ceph-mon[25702]: mds.? [v2:172.21.15.45:6834/558522566,v1:172.21.15.45:6835/558522566] up:active 2024-09-17T11:55:28.449 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:28 smithi165 ceph-mon[25702]: fsmap foofs:1 {0=foofs.smithi045.olwaas=up:active} 2024-09-17T11:55:30.125 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:55:30.126 INFO:teuthology.orchestra.run.smithi045.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-17T11:48:51.619752Z", "last_refresh": "2024-09-17T11:52:33.768295Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-17T11:48:45.016215Z", "last_refresh": "2024-09-17T11:52:33.768362Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-17T11:48:47.912727Z", "last_refresh": "2024-09-17T11:52:33.768426Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-17T11:55:22.384110Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-17T11:55:22.373606Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-17T11:48:43.055030Z", "last_refresh": "2024-09-17T11:52:33.768221Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi045:172.21.15.45=smithi045", "smithi165:172.21.15.165=smithi165"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-17T11:49:34.840638Z", "last_refresh": "2024-09-17T11:52:33.768076Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-17T11:48:50.361555Z", "last_refresh": "2024-09-17T11:52:33.768489Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-17T11:52:33.768613Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-17T11:48:46.212310Z", "last_refresh": "2024-09-17T11:52:33.768551Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-17T11:55:30.303 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:30 smithi165 ceph-mon[25702]: pgmap v221: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 3 op/s 2024-09-17T11:55:30.303 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:30 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:30.303 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:30 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:55:30.303 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:30 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:30.303 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:30 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:55:30.303 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:30 smithi165 ceph-mon[25702]: mds.? [v2:172.21.15.165:6832/1936605703,v1:172.21.15.165:6833/1936605703] up:boot 2024-09-17T11:55:30.304 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:30 smithi165 ceph-mon[25702]: fsmap foofs:1 {0=foofs.smithi045.olwaas=up:active} 1 up:standby 2024-09-17T11:55:30.304 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:30 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi165.czpmgd"}]: dispatch 2024-09-17T11:55:30.304 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:30 smithi165 ceph-mon[25702]: fsmap foofs:1 {0=foofs.smithi045.olwaas=up:active} 1 up:standby 2024-09-17T11:55:30.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:30 smithi045 ceph-mon[23737]: pgmap v221: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 3 op/s 2024-09-17T11:55:30.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:30 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:30.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:30 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:55:30.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:30 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:30.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:30 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:55:30.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:30 smithi045 ceph-mon[23737]: mds.? [v2:172.21.15.165:6832/1936605703,v1:172.21.15.165:6833/1936605703] up:boot 2024-09-17T11:55:30.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:30 smithi045 ceph-mon[23737]: fsmap foofs:1 {0=foofs.smithi045.olwaas=up:active} 1 up:standby 2024-09-17T11:55:30.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:30 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi165.czpmgd"}]: dispatch 2024-09-17T11:55:30.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:30 smithi045 ceph-mon[23737]: fsmap foofs:1 {0=foofs.smithi045.olwaas=up:active} 1 up:standby 2024-09-17T11:55:30.736 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-17T11:55:31.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:31 smithi165 ceph-mon[25702]: from='client.14496 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T11:55:31.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:31 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:31.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:31 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:31.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:31 smithi045 ceph-mon[23737]: from='client.14496 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T11:55:31.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:31 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:31.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:31 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:31.737 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-17T11:55:32.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:32 smithi045 ceph-mon[23737]: pgmap v222: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-17T11:55:32.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:32 smithi165 ceph-mon[25702]: pgmap v222: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-17T11:55:33.892 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:55:33.892 INFO:teuthology.orchestra.run.smithi045.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-17T11:48:51.619752Z", "last_refresh": "2024-09-17T11:52:33.768295Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-17T11:48:45.016215Z", "last_refresh": "2024-09-17T11:52:33.768362Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-17T11:48:47.912727Z", "last_refresh": "2024-09-17T11:52:33.768426Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-17T11:55:22.384110Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-17T11:55:22.373606Z", "last_refresh": "2024-09-17T11:55:31.004259Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-17T11:48:43.055030Z", "last_refresh": "2024-09-17T11:52:33.768221Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi045:172.21.15.45=smithi045", "smithi165:172.21.15.165=smithi165"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-17T11:49:34.840638Z", "last_refresh": "2024-09-17T11:52:33.768076Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-17T11:48:50.361555Z", "last_refresh": "2024-09-17T11:52:33.768489Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-17T11:52:33.768613Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-17T11:48:46.212310Z", "last_refresh": "2024-09-17T11:52:33.768551Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-17T11:55:34.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:34 smithi045 ceph-mon[23737]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-17T11:55:34.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:34 smithi165 ceph-mon[25702]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-17T11:55:35.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:35 smithi045 ceph-mon[23737]: from='client.14500 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T11:55:35.462 INFO:tasks.cephadm:mds.foofs has 1/2 2024-09-17T11:55:35.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:35 smithi165 ceph-mon[25702]: from='client.14500 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T11:55:36.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:36 smithi045 ceph-mon[23737]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-17T11:55:36.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:36 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:36.463 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-17T11:55:36.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:36 smithi165 ceph-mon[25702]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-17T11:55:36.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:36 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:37.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:37 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:37.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:37 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:37.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:37 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:55:37.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:37 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:37.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:37 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:55:37.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:37 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:37.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:37 smithi165 ceph-mon[25702]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-17T11:55:37.780 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:37 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:37.780 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:37 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:37.780 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:37 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:55:37.780 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:37 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:37.780 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:37 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:55:37.781 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:37 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:37.781 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:37 smithi045 ceph-mon[23737]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-17T11:55:38.520 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:55:38.520 INFO:teuthology.orchestra.run.smithi045.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-17T11:48:51.619752Z", "last_refresh": "2024-09-17T11:55:36.431194Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-17T11:48:45.016215Z", "last_refresh": "2024-09-17T11:55:31.003588Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-17T11:48:47.912727Z", "last_refresh": "2024-09-17T11:55:36.431325Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-17T11:55:22.384110Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-17T11:55:22.373606Z", "last_refresh": "2024-09-17T11:55:31.004259Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-17T11:48:43.055030Z", "last_refresh": "2024-09-17T11:55:31.003743Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi045:172.21.15.45=smithi045", "smithi165:172.21.15.165=smithi165"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-17T11:49:34.840638Z", "last_refresh": "2024-09-17T11:55:31.003822Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-17T11:48:50.361555Z", "last_refresh": "2024-09-17T11:55:31.003894Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-17T11:55:31.003963Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-17T11:48:46.212310Z", "last_refresh": "2024-09-17T11:55:36.431452Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-17T11:55:39.099 INFO:tasks.cephadm:mds.foofs has 2/2 2024-09-17T11:55:39.099 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T11:55:39.109 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi045.front.sepia.ceph.com 2024-09-17T11:55:39.109 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-09-17T11:55:40.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:39 smithi165 ceph-mon[25702]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T11:55:40.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:39 smithi165 ceph-mon[25702]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-17T11:55:40.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:39 smithi045 ceph-mon[23737]: from='client.14504 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T11:55:40.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:39 smithi045 ceph-mon[23737]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-17T11:55:42.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:41 smithi165 ceph-mon[25702]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 426 B/s wr, 1 op/s 2024-09-17T11:55:42.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:41 smithi165 ceph-mon[25702]: from='client.14508 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:42.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:41 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-17T11:55:42.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:41 smithi045 ceph-mon[23737]: pgmap v227: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 426 B/s wr, 1 op/s 2024-09-17T11:55:42.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:41 smithi045 ceph-mon[23737]: from='client.14508 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:42.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:41 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-17T11:55:42.777 INFO:teuthology.orchestra.run.smithi045.stdout:NFS Cluster Created Successfully 2024-09-17T11:55:43.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:42 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-17T11:55:43.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:42 smithi165 ceph-mon[25702]: osdmap e65: 8 total, 8 up, 8 in 2024-09-17T11:55:43.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:42 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-17T11:55:43.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:42 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-17T11:55:43.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:42 smithi045 ceph-mon[23737]: osdmap e65: 8 total, 8 up, 8 in 2024-09-17T11:55:43.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:42 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-17T11:55:43.468 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-09-17T11:55:44.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: pgmap v229: 97 pgs: 1 creating+peering, 31 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-17T11:55:44.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-17T11:55:44.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: osdmap e66: 8 total, 8 up, 8 in 2024-09-17T11:55:44.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: Saving service nfs.foo spec with placement count:2 2024-09-17T11:55:44.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:44.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:55:44.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:44.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:55:44.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:44.018 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:44.018 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: Creating key for client.nfs.foo.0.0.smithi045.qochkw 2024-09-17T11:55:44.018 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi045.qochkw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T11:55:44.018 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi045.qochkw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-17T11:55:44.018 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-17T11:55:44.018 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T11:55:44.019 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T11:55:44.019 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:44.019 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T11:55:44.019 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T11:55:44.019 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: Rados config object exists: conf-nfs.foo 2024-09-17T11:55:44.019 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: Creating key for client.nfs.foo.0.0.smithi045.qochkw-rgw 2024-09-17T11:55:44.019 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi045.qochkw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T11:55:44.019 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi045.qochkw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T11:55:44.020 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:44.020 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:43 smithi165 ceph-mon[25702]: Deploying daemon nfs.foo.0.0.smithi045.qochkw on smithi045 2024-09-17T11:55:44.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: pgmap v229: 97 pgs: 1 creating+peering, 31 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-17T11:55:44.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-17T11:55:44.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: osdmap e66: 8 total, 8 up, 8 in 2024-09-17T11:55:44.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: Saving service nfs.foo spec with placement count:2 2024-09-17T11:55:44.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:44.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:55:44.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:44.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:55:44.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:44.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:44.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: Creating key for client.nfs.foo.0.0.smithi045.qochkw 2024-09-17T11:55:44.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi045.qochkw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T11:55:44.092 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi045.qochkw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-17T11:55:44.092 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-17T11:55:44.092 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T11:55:44.092 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T11:55:44.092 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:44.092 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T11:55:44.092 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T11:55:44.092 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: Rados config object exists: conf-nfs.foo 2024-09-17T11:55:44.092 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: Creating key for client.nfs.foo.0.0.smithi045.qochkw-rgw 2024-09-17T11:55:44.092 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi045.qochkw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T11:55:44.092 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi045.qochkw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T11:55:44.092 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:44.092 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:43 smithi045 ceph-mon[23737]: Deploying daemon nfs.foo.0.0.smithi045.qochkw on smithi045 2024-09-17T11:55:44.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:44 smithi045 ceph-mon[23737]: osdmap e67: 8 total, 8 up, 8 in 2024-09-17T11:55:45.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:44 smithi165 ceph-mon[25702]: osdmap e67: 8 total, 8 up, 8 in 2024-09-17T11:55:46.000 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:45 smithi045 ceph-mon[23737]: pgmap v232: 97 pgs: 5 creating+activating, 1 creating+peering, 10 unknown, 81 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:46.000 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:45 smithi045 ceph-mon[23737]: mgrmap e20: smithi045.jtwyrp(active, since 5m), standbys: smithi165.ykdszc 2024-09-17T11:55:46.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:45 smithi165 ceph-mon[25702]: pgmap v232: 97 pgs: 5 creating+activating, 1 creating+peering, 10 unknown, 81 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-17T11:55:46.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:45 smithi165 ceph-mon[25702]: mgrmap e20: smithi045.jtwyrp(active, since 5m), standbys: smithi165.ykdszc 2024-09-17T11:55:46.699 INFO:teuthology.orchestra.run.smithi045.stderr:Invalid command: Unexpected argument '--clusterid' 2024-09-17T11:55:46.699 INFO:teuthology.orchestra.run.smithi045.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-09-17T11:55:46.699 INFO:teuthology.orchestra.run.smithi045.stderr:Error EINVAL: invalid command 2024-09-17T11:55:47.071 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T11:55:47.072 INFO:teuthology.orchestra.run.smithi045.stdout: "bind": "/fake", 2024-09-17T11:55:47.072 INFO:teuthology.orchestra.run.smithi045.stdout: "fs": "foofs", 2024-09-17T11:55:47.072 INFO:teuthology.orchestra.run.smithi045.stdout: "path": "/", 2024-09-17T11:55:47.072 INFO:teuthology.orchestra.run.smithi045.stdout: "cluster": "foo", 2024-09-17T11:55:47.072 INFO:teuthology.orchestra.run.smithi045.stdout: "mode": "RW" 2024-09-17T11:55:47.072 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T11:55:47.839 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-09-17T11:55:48.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:48 smithi045 ceph-mon[23737]: pgmap v233: 97 pgs: 5 creating+activating, 1 creating+peering, 91 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-17T11:55:48.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:48 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:48.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:48 smithi045 ceph-mon[23737]: Creating key for client.nfs.foo.1.0.smithi165.cvmryy 2024-09-17T11:55:48.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:48 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi165.cvmryy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T11:55:48.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:48 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi165.cvmryy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-17T11:55:48.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:48 smithi045 ceph-mon[23737]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-17T11:55:48.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:48 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T11:55:48.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:48 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T11:55:48.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:48 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:48.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:48 smithi045 ceph-mon[23737]: from='client.14528 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:48.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:48 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-17T11:55:48.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:48 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-17T11:55:48.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:48 smithi165 ceph-mon[25702]: pgmap v233: 97 pgs: 5 creating+activating, 1 creating+peering, 91 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-17T11:55:48.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:48 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:48.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:48 smithi165 ceph-mon[25702]: Creating key for client.nfs.foo.1.0.smithi165.cvmryy 2024-09-17T11:55:48.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:48 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi165.cvmryy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T11:55:48.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:48 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi165.cvmryy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-17T11:55:48.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:48 smithi165 ceph-mon[25702]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-17T11:55:48.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:48 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T11:55:48.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:48 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T11:55:48.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:48 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:48.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:48 smithi165 ceph-mon[25702]: from='client.14528 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:48.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:48 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-17T11:55:48.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:48 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-17T11:55:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:50 smithi165 ceph-mon[25702]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 148 B/s rd, 296 B/s wr, 0 op/s 2024-09-17T11:55:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:50 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T11:55:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:50 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T11:55:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:50 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi165.cvmryy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T11:55:50.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:50 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi165.cvmryy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T11:55:50.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:50 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:50.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:50 smithi045 ceph-mon[23737]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 148 B/s rd, 296 B/s wr, 0 op/s 2024-09-17T11:55:50.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:50 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T11:55:50.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:50 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T11:55:50.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:50 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi165.cvmryy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T11:55:50.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:50 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi165.cvmryy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T11:55:50.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:50 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:51.290 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:51 smithi165 ceph-mon[25702]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:51.290 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:51 smithi165 ceph-mon[25702]: Rados config object exists: conf-nfs.foo 2024-09-17T11:55:51.290 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:51 smithi165 ceph-mon[25702]: Creating key for client.nfs.foo.1.0.smithi165.cvmryy-rgw 2024-09-17T11:55:51.290 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:51 smithi165 ceph-mon[25702]: Deploying daemon nfs.foo.1.0.smithi165.cvmryy on smithi165 2024-09-17T11:55:51.291 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:51 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]: dispatch 2024-09-17T11:55:51.291 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:51 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-09-17T11:55:51.291 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:51 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-09-17T11:55:51.291 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:51 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-09-17T11:55:51.291 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:51 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:55:51.291 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:51 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:55:51.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:51 smithi045 ceph-mon[23737]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:51.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:51 smithi045 ceph-mon[23737]: Rados config object exists: conf-nfs.foo 2024-09-17T11:55:51.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:51 smithi045 ceph-mon[23737]: Creating key for client.nfs.foo.1.0.smithi165.cvmryy-rgw 2024-09-17T11:55:51.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:51 smithi045 ceph-mon[23737]: Deploying daemon nfs.foo.1.0.smithi165.cvmryy on smithi165 2024-09-17T11:55:51.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]: dispatch 2024-09-17T11:55:51.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-09-17T11:55:51.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-09-17T11:55:51.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]: dispatch 2024-09-17T11:55:51.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:55:51.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:55:52.315 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:52 smithi165 ceph-mon[25702]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 255 B/s wr, 0 op/s 2024-09-17T11:55:52.315 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:52 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]': finished 2024-09-17T11:55:52.315 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:52 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-09-17T11:55:52.315 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:52 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-09-17T11:55:52.315 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:52 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-09-17T11:55:52.316 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:52 smithi165 ceph-mon[25702]: osdmap e68: 8 total, 8 up, 8 in 2024-09-17T11:55:52.316 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:52 smithi165 ceph-mon[25702]: from='client.14558 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:52.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:52 smithi045 ceph-mon[23737]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 255 B/s wr, 0 op/s 2024-09-17T11:55:52.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:52 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1f", "id": [7, 2]}]': finished 2024-09-17T11:55:52.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:52 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-09-17T11:55:52.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:52 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-09-17T11:55:52.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:52 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-09-17T11:55:52.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:52 smithi045 ceph-mon[23737]: osdmap e68: 8 total, 8 up, 8 in 2024-09-17T11:55:52.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:52 smithi045 ceph-mon[23737]: from='client.14558 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:53.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:53 smithi165 ceph-mon[25702]: osdmap e69: 8 total, 8 up, 8 in 2024-09-17T11:55:53.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:53 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:53.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:53 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:55:53.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:53 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:53.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:53 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:55:53.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:53 smithi045 ceph-mon[23737]: osdmap e69: 8 total, 8 up, 8 in 2024-09-17T11:55:53.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:53 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:53.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:53 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:55:53.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:53 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:53.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:53 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:55:54.514 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:54 smithi045 ceph-mon[23737]: from='client.14568 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:54.514 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:54 smithi045 ceph-mon[23737]: pgmap v238: 97 pgs: 2 peering, 95 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1.9 KiB/s wr, 4 op/s 2024-09-17T11:55:54.612 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:54 smithi165 ceph-mon[25702]: from='client.14568 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:54.612 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:54 smithi165 ceph-mon[25702]: pgmap v238: 97 pgs: 2 peering, 95 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1.9 KiB/s wr, 4 op/s 2024-09-17T11:55:55.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:55 smithi045 ceph-mon[23737]: from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:55.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:55 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:55.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:55 smithi165 ceph-mon[25702]: from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:55.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:55 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:56.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:56 smithi045 ceph-mon[23737]: pgmap v239: 97 pgs: 2 peering, 95 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.6 KiB/s wr, 3 op/s; 69 B/s, 0 objects/s recovering 2024-09-17T11:55:56.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:56 smithi045 ceph-mon[23737]: from='client.14576 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:56.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:56 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:56.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:56 smithi165 ceph-mon[25702]: pgmap v239: 97 pgs: 2 peering, 95 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1.6 KiB/s wr, 3 op/s; 69 B/s, 0 objects/s recovering 2024-09-17T11:55:56.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:56 smithi165 ceph-mon[25702]: from='client.14576 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:56.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:56 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:57.798 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo ?:2049 2/2 2s ago 15s count:2 2024-09-17T11:55:58.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:58 smithi045 ceph-mon[23737]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:58.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:58 smithi045 ceph-mon[23737]: pgmap v240: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s; 69 B/s, 0 objects/s recovering 2024-09-17T11:55:58.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:58 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:58.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:58 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:58.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:58 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:58.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:58 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:55:58.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:58 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:58.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:58 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:55:58.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:58 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:58.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:58 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:58.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:58 smithi165 ceph-mon[25702]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:58.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:58 smithi165 ceph-mon[25702]: pgmap v240: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s; 69 B/s, 0 objects/s recovering 2024-09-17T11:55:58.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:58 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:58.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:58 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:58.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:58 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:58.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:58 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:55:58.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:58 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:55:58.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:58 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:55:58.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:58 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:58.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:58 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:55:58.502 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-17T11:55:58.512 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi045.front.sepia.ceph.com 2024-09-17T11:55:58.512 DEBUG:teuthology.orchestra.run.smithi045:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-09-17T11:55:58.539 INFO:teuthology.orchestra.run.smithi045.stderr:+ mkdir /mnt/foo 2024-09-17T11:55:58.565 DEBUG:teuthology.orchestra.run.smithi045:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-09-17T11:55:58.629 INFO:teuthology.orchestra.run.smithi045.stderr:++ hostname 2024-09-17T11:55:58.630 INFO:teuthology.orchestra.run.smithi045.stderr:+ mount -t nfs smithi045:/fake /mnt/foo -o sync 2024-09-17T11:55:59.073 DEBUG:teuthology.orchestra.run.smithi045:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-09-17T11:55:59.140 INFO:teuthology.orchestra.run.smithi045.stderr:+ echo test 2024-09-17T11:55:59.159 DEBUG:teuthology.orchestra.run.smithi045:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-09-17T11:55:59.221 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:55:59 smithi045 ceph-mon[23737]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:59.224 INFO:teuthology.orchestra.run.smithi045.stderr:+ sync 2024-09-17T11:55:59.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:55:59 smithi165 ceph-mon[25702]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:55:59.484 INFO:teuthology.run_tasks:Running task parallel... 2024-09-17T11:55:59.495 INFO:teuthology.task.parallel:starting parallel... 2024-09-17T11:55:59.495 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-17T11:55:59.496 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-17T11:55:59.496 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi045.front.sepia.ceph.com 2024-09-17T11:55:59.497 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-17T11:55:59.497 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-17T11:55:59.497 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-09-17T11:55:59.498 INFO:teuthology.task.exec:Executing custom commands... 2024-09-17T11:55:59.498 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi045.front.sepia.ceph.com 2024-09-17T11:55:59.499 DEBUG:teuthology.orchestra.run.smithi045:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-09-17T11:55:59.568 INFO:teuthology.orchestra.run.smithi045.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-17T11:55:59.568 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T11:55:59.568 INFO:teuthology.orchestra.run.smithi045.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-09-17T11:55:59.569 INFO:teuthology.orchestra.run.smithi045.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-17T11:55:59.669 INFO:teuthology.orchestra.run.smithi045.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-17T11:55:59.669 INFO:teuthology.orchestra.run.smithi045.stdout:releasing clients 2024-09-17T11:56:00.092 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:00 smithi045 ceph-mon[23737]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s; 69 B/s, 0 objects/s recovering 2024-09-17T11:56:00.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:00 smithi165 ceph-mon[25702]: pgmap v241: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s; 69 B/s, 0 objects/s recovering 2024-09-17T11:56:00.669 INFO:teuthology.orchestra.run.smithi045.stdout: 5 125 28.04 MB/sec warmup 1 sec latency 31.898 ms 2024-09-17T11:56:01.669 INFO:teuthology.orchestra.run.smithi045.stdout: 5 255 24.06 MB/sec warmup 2 sec latency 32.617 ms 2024-09-17T11:56:02.175 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:02 smithi045 ceph-mon[23737]: pgmap v242: 97 pgs: 97 active+clean; 1.6 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.8 KiB/s rd, 108 KiB/s wr, 6 op/s; 57 B/s, 0 objects/s recovering 2024-09-17T11:56:02.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:02 smithi165 ceph-mon[25702]: pgmap v242: 97 pgs: 97 active+clean; 1.6 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.8 KiB/s rd, 108 KiB/s wr, 6 op/s; 57 B/s, 0 objects/s recovering 2024-09-17T11:56:02.669 INFO:teuthology.orchestra.run.smithi045.stdout: 5 386 22.89 MB/sec warmup 3 sec latency 38.328 ms 2024-09-17T11:56:03.121 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-17T11:56:03.669 INFO:teuthology.orchestra.run.smithi045.stdout: 5 518 22.45 MB/sec warmup 4 sec latency 31.609 ms 2024-09-17T11:56:04.254 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:04 smithi045 ceph-mon[23737]: pgmap v243: 97 pgs: 97 active+clean; 15 MiB data, 56 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 1.5 MiB/s wr, 80 op/s; 52 B/s, 0 objects/s recovering 2024-09-17T11:56:04.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:04 smithi165 ceph-mon[25702]: pgmap v243: 97 pgs: 97 active+clean; 15 MiB data, 56 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 1.5 MiB/s wr, 80 op/s; 52 B/s, 0 objects/s recovering 2024-09-17T11:56:04.669 INFO:teuthology.orchestra.run.smithi045.stdout: 5 748 21.33 MB/sec warmup 5 sec latency 32.375 ms 2024-09-17T11:56:05.669 INFO:teuthology.orchestra.run.smithi045.stdout: 5 1107 19.12 MB/sec warmup 6 sec latency 30.993 ms 2024-09-17T11:56:06.043 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:06 smithi045 ceph-mon[23737]: pgmap v244: 97 pgs: 97 active+clean; 52 MiB data, 90 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 4.4 MiB/s wr, 130 op/s; 46 B/s, 0 objects/s recovering 2024-09-17T11:56:06.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:06 smithi165 ceph-mon[25702]: pgmap v244: 97 pgs: 97 active+clean; 52 MiB data, 90 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 4.4 MiB/s wr, 130 op/s; 46 B/s, 0 objects/s recovering 2024-09-17T11:56:06.669 INFO:teuthology.orchestra.run.smithi045.stdout: 5 1844 18.82 MB/sec warmup 7 sec latency 30.629 ms 2024-09-17T11:56:07.249 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-17T11:56:07.669 INFO:teuthology.orchestra.run.smithi045.stdout: 5 2498 17.91 MB/sec warmup 8 sec latency 68.837 ms 2024-09-17T11:56:08.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:08 smithi045 ceph-mon[23737]: pgmap v245: 97 pgs: 97 active+clean; 95 MiB data, 181 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 8.3 MiB/s wr, 401 op/s 2024-09-17T11:56:08.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:08 smithi165 ceph-mon[25702]: pgmap v245: 97 pgs: 97 active+clean; 95 MiB data, 181 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 8.3 MiB/s wr, 401 op/s 2024-09-17T11:56:08.669 INFO:teuthology.orchestra.run.smithi045.stdout: 5 3165 19.00 MB/sec warmup 9 sec latency 48.752 ms 2024-09-17T11:56:09.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 3662 19.16 MB/sec warmup 10 sec latency 50.924 ms 2024-09-17T11:56:10.121 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:10 smithi045 ceph-mon[23737]: pgmap v246: 97 pgs: 97 active+clean; 112 MiB data, 211 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 10 MiB/s wr, 460 op/s 2024-09-17T11:56:10.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:10 smithi165 ceph-mon[25702]: pgmap v246: 97 pgs: 97 active+clean; 112 MiB data, 211 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 10 MiB/s wr, 460 op/s 2024-09-17T11:56:10.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 3915 18.18 MB/sec warmup 11 sec latency 42.443 ms 2024-09-17T11:56:11.407 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph mgr module enable nfs --force' 2024-09-17T11:56:11.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 4440 16.97 MB/sec warmup 12 sec latency 83.797 ms 2024-09-17T11:56:12.176 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:12 smithi045 ceph-mon[23737]: pgmap v247: 97 pgs: 97 active+clean; 119 MiB data, 218 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 11 MiB/s wr, 600 op/s 2024-09-17T11:56:12.317 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:12 smithi165 ceph-mon[25702]: pgmap v247: 97 pgs: 97 active+clean; 119 MiB data, 218 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 11 MiB/s wr, 600 op/s 2024-09-17T11:56:12.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 4972 17.13 MB/sec warmup 13 sec latency 64.106 ms 2024-09-17T11:56:13.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 5628 16.39 MB/sec warmup 14 sec latency 32.758 ms 2024-09-17T11:56:14.197 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:14 smithi045 ceph-mon[23737]: pgmap v248: 97 pgs: 97 active+clean; 125 MiB data, 207 MiB used, 715 GiB / 715 GiB avail; 3.3 KiB/s rd, 12 MiB/s wr, 627 op/s 2024-09-17T11:56:14.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:14 smithi165 ceph-mon[25702]: pgmap v248: 97 pgs: 97 active+clean; 125 MiB data, 207 MiB used, 715 GiB / 715 GiB avail; 3.3 KiB/s rd, 12 MiB/s wr, 627 op/s 2024-09-17T11:56:14.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 6011 15.78 MB/sec warmup 15 sec latency 46.322 ms 2024-09-17T11:56:15.075 INFO:teuthology.orchestra.run.smithi045.stderr:module 'nfs' is already enabled 2024-09-17T11:56:15.228 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:15 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/835322283' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-17T11:56:15.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:15 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/835322283' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-17T11:56:15.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 6533 16.27 MB/sec warmup 16 sec latency 71.715 ms 2024-09-17T11:56:16.241 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:16 smithi045 ceph-mon[23737]: pgmap v249: 97 pgs: 97 active+clean; 131 MiB data, 240 MiB used, 715 GiB / 715 GiB avail; 3.3 KiB/s rd, 12 MiB/s wr, 687 op/s 2024-09-17T11:56:16.242 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:16 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/835322283' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-17T11:56:16.242 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:16 smithi045 ceph-mon[23737]: mgrmap e21: smithi045.jtwyrp(active, since 6m), standbys: smithi165.ykdszc 2024-09-17T11:56:16.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:16 smithi165 ceph-mon[25702]: pgmap v249: 97 pgs: 97 active+clean; 131 MiB data, 240 MiB used, 715 GiB / 715 GiB avail; 3.3 KiB/s rd, 12 MiB/s wr, 687 op/s 2024-09-17T11:56:16.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:16 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/835322283' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-17T11:56:16.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:16 smithi165 ceph-mon[25702]: mgrmap e21: smithi045.jtwyrp(active, since 6m), standbys: smithi165.ykdszc 2024-09-17T11:56:16.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 6951 16.30 MB/sec warmup 17 sec latency 55.489 ms 2024-09-17T11:56:16.873 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-17T11:56:17.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 7242 16.10 MB/sec warmup 18 sec latency 78.284 ms 2024-09-17T11:56:18.313 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:18 smithi045 ceph-mon[23737]: pgmap v250: 97 pgs: 97 active+clean; 134 MiB data, 287 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 11 MiB/s wr, 855 op/s 2024-09-17T11:56:18.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:18 smithi165 ceph-mon[25702]: pgmap v250: 97 pgs: 97 active+clean; 134 MiB data, 287 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 11 MiB/s wr, 855 op/s 2024-09-17T11:56:18.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 7425 15.53 MB/sec warmup 19 sec latency 73.201 ms 2024-09-17T11:56:19.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 7740 14.84 MB/sec warmup 20 sec latency 125.000 ms 2024-09-17T11:56:19.717 INFO:teuthology.orchestra.run.smithi045.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:7498493279982dfe87f00616198a5967475f6169 2024-09-17T11:56:20.336 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:20 smithi045 ceph-mon[23737]: pgmap v251: 97 pgs: 97 active+clean; 135 MiB data, 316 MiB used, 715 GiB / 715 GiB avail; 8.1 KiB/s rd, 8.1 MiB/s wr, 631 op/s 2024-09-17T11:56:20.336 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:20 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:56:20.336 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:20 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:56:20.336 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:20 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:56:20.336 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:20 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:56:20.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:20 smithi165 ceph-mon[25702]: pgmap v251: 97 pgs: 97 active+clean; 135 MiB data, 316 MiB used, 715 GiB / 715 GiB avail; 8.1 KiB/s rd, 8.1 MiB/s wr, 631 op/s 2024-09-17T11:56:20.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:20 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:56:20.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:20 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:56:20.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:20 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:56:20.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:20 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:56:20.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 8043 14.31 MB/sec warmup 21 sec latency 77.532 ms 2024-09-17T11:56:20.987 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-17T11:56:20.988 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi045.front.sepia.ceph.com 2024-09-17T11:56:20.988 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-17T11:56:21.377 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:21 smithi045 ceph-mon[23737]: from='client.14604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:7498493279982dfe87f00616198a5967475f6169", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:56:21.381 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:21 smithi045 ceph-mon[23737]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:7498493279982dfe87f00616198a5967475f6169 2024-09-17T11:56:21.381 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:21 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:56:21.381 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:21 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:56:21.381 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:21 smithi045 ceph-mon[23737]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:7498493279982dfe87f00616198a5967475f6169 2024-09-17T11:56:21.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 8262 13.89 MB/sec warmup 22 sec latency 82.153 ms 2024-09-17T11:56:21.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:21 smithi165 ceph-mon[25702]: from='client.14604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:7498493279982dfe87f00616198a5967475f6169", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:56:21.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:21 smithi165 ceph-mon[25702]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:7498493279982dfe87f00616198a5967475f6169 2024-09-17T11:56:21.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:21 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:56:21.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:21 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:56:21.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:21 smithi165 ceph-mon[25702]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:7498493279982dfe87f00616198a5967475f6169 2024-09-17T11:56:22.328 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:22 smithi045 ceph-mon[23737]: pgmap v252: 97 pgs: 97 active+clean; 145 MiB data, 339 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 7.5 MiB/s wr, 709 op/s 2024-09-17T11:56:22.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 8631 13.86 MB/sec warmup 23 sec latency 40.113 ms 2024-09-17T11:56:22.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:22 smithi165 ceph-mon[25702]: pgmap v252: 97 pgs: 97 active+clean; 145 MiB data, 339 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 7.5 MiB/s wr, 709 op/s 2024-09-17T11:56:23.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 9130 13.50 MB/sec warmup 24 sec latency 59.090 ms 2024-09-17T11:56:24.241 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T11:56:24.555 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:24 smithi045 ceph-mon[23737]: pgmap v253: 97 pgs: 97 active+clean; 146 MiB data, 368 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.4 MiB/s wr, 586 op/s 2024-09-17T11:56:24.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 9481 13.22 MB/sec warmup 25 sec latency 62.225 ms 2024-09-17T11:56:24.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:24 smithi165 ceph-mon[25702]: pgmap v253: 97 pgs: 97 active+clean; 146 MiB data, 368 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.4 MiB/s wr, 586 op/s 2024-09-17T11:56:25.226 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T11:56:25.226 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (5m) 28s ago 6m 22.2M - ba2b418f427c c97407c70fb6 2024-09-17T11:56:25.226 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (6m) 28s ago 6m 6413k - 17.2.0 e1d6a67b021e 8c510943b55e 2024-09-17T11:56:25.226 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (6m) 30s ago 6m 6409k - 17.2.0 e1d6a67b021e 687c24a87b70 2024-09-17T11:56:25.226 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (5m) 28s ago 6m 53.5M - 8.3.5 dad864ee21e9 7d87d4101e56 2024-09-17T11:56:25.226 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (59s) 28s ago 58s 13.7M - 17.2.0 e1d6a67b021e 6a2a4a998dd1 2024-09-17T11:56:25.226 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (56s) 30s ago 56s 13.3M - 17.2.0 e1d6a67b021e d4c2987b9bac 2024-09-17T11:56:25.226 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:9283 running (8m) 28s ago 8m 460M - 17.2.0 e1d6a67b021e 5f496de39b08 2024-09-17T11:56:25.226 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283 running (6m) 30s ago 6m 411M - 17.2.0 e1d6a67b021e fefbdcc12a47 2024-09-17T11:56:25.226 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (8m) 28s ago 8m 47.9M 2048M 17.2.0 e1d6a67b021e c4256bdf51b7 2024-09-17T11:56:25.226 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (6m) 30s ago 6m 34.7M 2048M 17.2.0 e1d6a67b021e 9219263d3aa8 2024-09-17T11:56:25.227 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (38s) 28s ago 38s 49.0M - 4.0 e1d6a67b021e b8fc749ebb71 2024-09-17T11:56:25.227 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (33s) 30s ago 32s 48.8M - 4.0 e1d6a67b021e 771b27764432 2024-09-17T11:56:25.227 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (6m) 28s ago 6m 17.5M - 1dbe0e931976 09dea232cb6b 2024-09-17T11:56:25.227 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (6m) 30s ago 6m 16.6M - 1dbe0e931976 fdb9449adb13 2024-09-17T11:56:25.227 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (5m) 28s ago 5m 42.5M 3217M 17.2.0 e1d6a67b021e e0ce4d29dc64 2024-09-17T11:56:25.227 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (5m) 28s ago 5m 43.7M 3217M 17.2.0 e1d6a67b021e da8bda7b47b3 2024-09-17T11:56:25.227 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (4m) 28s ago 4m 40.4M 3217M 17.2.0 e1d6a67b021e 21fe0d9e5ced 2024-09-17T11:56:25.227 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (4m) 28s ago 4m 45.3M 3217M 17.2.0 e1d6a67b021e 92a2389488e4 2024-09-17T11:56:25.227 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (3m) 30s ago 3m 42.1M 3984M 17.2.0 e1d6a67b021e e29eb74df25a 2024-09-17T11:56:25.227 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (3m) 30s ago 3m 40.4M 3984M 17.2.0 e1d6a67b021e 5d900eb9165b 2024-09-17T11:56:25.227 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (2m) 30s ago 2m 42.0M 3984M 17.2.0 e1d6a67b021e fae3e1bae466 2024-09-17T11:56:25.227 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (2m) 30s ago 2m 39.6M 3984M 17.2.0 e1d6a67b021e 71b9f2cdb810 2024-09-17T11:56:25.227 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (5m) 28s ago 5m 60.0M - 514e6a882f6e 528c529e18d6 2024-09-17T11:56:25.570 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:25 smithi045 ceph-mon[23737]: from='client.24397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:56:25.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 9886 13.45 MB/sec warmup 26 sec latency 42.864 ms 2024-09-17T11:56:25.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:25 smithi165 ceph-mon[25702]: from='client.24397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:56:25.766 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T11:56:25.766 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T11:56:25.766 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T11:56:25.766 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:56:25.766 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T11:56:25.766 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T11:56:25.767 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:56:25.767 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T11:56:25.767 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T11:56:25.767 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:56:25.767 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T11:56:25.767 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T11:56:25.767 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:56:25.767 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T11:56:25.767 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-17T11:56:25.767 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T11:56:25.767 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T11:56:26.192 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T11:56:26.192 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:7498493279982dfe87f00616198a5967475f6169", 2024-09-17T11:56:26.192 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T11:56:26.192 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [], 2024-09-17T11:56:26.192 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "", 2024-09-17T11:56:26.192 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:7498493279982dfe87f00616198a5967475f6169 image" 2024-09-17T11:56:26.192 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T11:56:26.333 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:26 smithi045 ceph-mon[23737]: pgmap v254: 97 pgs: 97 active+clean; 139 MiB data, 406 MiB used, 715 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.1 MiB/s wr, 594 op/s 2024-09-17T11:56:26.333 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:26 smithi045 ceph-mon[23737]: from='client.14612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:56:26.333 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:26 smithi045 ceph-mon[23737]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:56:26.333 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:26 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/4277729648' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T11:56:26.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 10334 13.50 MB/sec warmup 27 sec latency 61.789 ms 2024-09-17T11:56:26.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:26 smithi165 ceph-mon[25702]: pgmap v254: 97 pgs: 97 active+clean; 139 MiB data, 406 MiB used, 715 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.1 MiB/s wr, 594 op/s 2024-09-17T11:56:26.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:26 smithi165 ceph-mon[25702]: from='client.14612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:56:26.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:26 smithi165 ceph-mon[25702]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:56:26.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:26 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/4277729648' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T11:56:26.791 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_OK 2024-09-17T11:56:27.356 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:27 smithi045 ceph-mon[23737]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:56:27.356 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:27 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/3597719218' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T11:56:27.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 10820 13.82 MB/sec warmup 28 sec latency 39.953 ms 2024-09-17T11:56:27.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:27 smithi165 ceph-mon[25702]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:56:27.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:27 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/3597719218' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T11:56:28.384 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:28 smithi045 ceph-mon[23737]: pgmap v255: 97 pgs: 97 active+clean; 140 MiB data, 409 MiB used, 715 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.8 MiB/s wr, 629 op/s 2024-09-17T11:56:28.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 11031 13.49 MB/sec warmup 29 sec latency 62.440 ms 2024-09-17T11:56:28.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:28 smithi165 ceph-mon[25702]: pgmap v255: 97 pgs: 97 active+clean; 140 MiB data, 409 MiB used, 715 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.8 MiB/s wr, 629 op/s 2024-09-17T11:56:29.408 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:29 smithi045 ceph-mon[23737]: pgmap v256: 97 pgs: 97 active+clean; 149 MiB data, 434 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 5.6 MiB/s wr, 529 op/s 2024-09-17T11:56:29.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 11393 13.11 MB/sec warmup 30 sec latency 65.294 ms 2024-09-17T11:56:29.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:29 smithi165 ceph-mon[25702]: pgmap v256: 97 pgs: 97 active+clean; 149 MiB data, 434 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 5.6 MiB/s wr, 529 op/s 2024-09-17T11:56:30.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 11654 12.83 MB/sec warmup 31 sec latency 65.477 ms 2024-09-17T11:56:31.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 11958 12.75 MB/sec warmup 32 sec latency 38.947 ms 2024-09-17T11:56:31.959 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:31 smithi045 ceph-mon[23737]: pgmap v257: 97 pgs: 97 active+clean; 154 MiB data, 433 MiB used, 715 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.2 MiB/s wr, 510 op/s 2024-09-17T11:56:32.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:31 smithi165 ceph-mon[25702]: pgmap v257: 97 pgs: 97 active+clean; 154 MiB data, 433 MiB used, 715 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.2 MiB/s wr, 510 op/s 2024-09-17T11:56:32.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 12412 12.67 MB/sec warmup 33 sec latency 57.744 ms 2024-09-17T11:56:33.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 12907 12.45 MB/sec warmup 34 sec latency 48.896 ms 2024-09-17T11:56:33.996 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:33 smithi045 ceph-mon[23737]: pgmap v258: 97 pgs: 97 active+clean; 155 MiB data, 443 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 4.4 MiB/s wr, 389 op/s 2024-09-17T11:56:34.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:33 smithi165 ceph-mon[25702]: pgmap v258: 97 pgs: 97 active+clean; 155 MiB data, 443 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 4.4 MiB/s wr, 389 op/s 2024-09-17T11:56:34.670 INFO:teuthology.orchestra.run.smithi045.stdout: 5 13154 12.27 MB/sec warmup 35 sec latency 42.983 ms 2024-09-17T11:56:35.671 INFO:teuthology.orchestra.run.smithi045.stdout: 5 13628 12.58 MB/sec warmup 36 sec latency 53.074 ms 2024-09-17T11:56:35.773 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:35 smithi045 ceph-mon[23737]: pgmap v259: 97 pgs: 97 active+clean; 151 MiB data, 480 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 481 op/s 2024-09-17T11:56:36.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:35 smithi165 ceph-mon[25702]: pgmap v259: 97 pgs: 97 active+clean; 151 MiB data, 480 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.9 MiB/s wr, 481 op/s 2024-09-17T11:56:36.671 INFO:teuthology.orchestra.run.smithi045.stdout: 5 14098 12.71 MB/sec warmup 37 sec latency 39.596 ms 2024-09-17T11:56:37.671 INFO:teuthology.orchestra.run.smithi045.stdout: 5 14480 12.79 MB/sec warmup 38 sec latency 36.993 ms 2024-09-17T11:56:37.788 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:37 smithi045 ceph-mon[23737]: pgmap v260: 97 pgs: 97 active+clean; 154 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.7 MiB/s wr, 583 op/s 2024-09-17T11:56:38.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:37 smithi165 ceph-mon[25702]: pgmap v260: 97 pgs: 97 active+clean; 154 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.7 MiB/s wr, 583 op/s 2024-09-17T11:56:38.671 INFO:teuthology.orchestra.run.smithi045.stdout: 5 15038 12.56 MB/sec warmup 39 sec latency 43.358 ms 2024-09-17T11:56:39.671 INFO:teuthology.orchestra.run.smithi045.stdout: 5 15253 12.35 MB/sec warmup 40 sec latency 69.087 ms 2024-09-17T11:56:39.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:39 smithi045 ceph-mon[23737]: pgmap v261: 97 pgs: 97 active+clean; 167 MiB data, 541 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.3 MiB/s wr, 590 op/s 2024-09-17T11:56:40.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:39 smithi165 ceph-mon[25702]: pgmap v261: 97 pgs: 97 active+clean; 167 MiB data, 541 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.3 MiB/s wr, 590 op/s 2024-09-17T11:56:40.671 INFO:teuthology.orchestra.run.smithi045.stdout: 5 15518 12.28 MB/sec warmup 41 sec latency 49.556 ms 2024-09-17T11:56:41.671 INFO:teuthology.orchestra.run.smithi045.stdout: 5 15943 12.23 MB/sec warmup 42 sec latency 54.301 ms 2024-09-17T11:56:41.899 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:41 smithi045 ceph-mon[23737]: pgmap v262: 97 pgs: 97 active+clean; 171 MiB data, 547 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.7 MiB/s wr, 512 op/s 2024-09-17T11:56:42.066 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:41 smithi165 ceph-mon[25702]: pgmap v262: 97 pgs: 97 active+clean; 171 MiB data, 547 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.7 MiB/s wr, 512 op/s 2024-09-17T11:56:42.671 INFO:teuthology.orchestra.run.smithi045.stdout: 5 16509 12.07 MB/sec warmup 43 sec latency 46.474 ms 2024-09-17T11:56:43.671 INFO:teuthology.orchestra.run.smithi045.stdout: 5 16943 12.19 MB/sec warmup 44 sec latency 32.757 ms 2024-09-17T11:56:43.953 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:43 smithi045 ceph-mon[23737]: pgmap v263: 97 pgs: 97 active+clean; 168 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 5.2 MiB/s wr, 504 op/s 2024-09-17T11:56:44.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:43 smithi165 ceph-mon[25702]: pgmap v263: 97 pgs: 97 active+clean; 168 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 5.2 MiB/s wr, 504 op/s 2024-09-17T11:56:44.671 INFO:teuthology.orchestra.run.smithi045.stdout: 5 17517 12.44 MB/sec warmup 45 sec latency 58.459 ms 2024-09-17T11:56:45.671 INFO:teuthology.orchestra.run.smithi045.stdout: 5 17919 12.57 MB/sec warmup 46 sec latency 70.038 ms 2024-09-17T11:56:46.008 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:45 smithi045 ceph-mon[23737]: pgmap v264: 97 pgs: 97 active+clean; 167 MiB data, 592 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.0 MiB/s wr, 631 op/s 2024-09-17T11:56:46.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:45 smithi165 ceph-mon[25702]: pgmap v264: 97 pgs: 97 active+clean; 167 MiB data, 592 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.0 MiB/s wr, 631 op/s 2024-09-17T11:56:46.671 INFO:teuthology.orchestra.run.smithi045.stdout: 5 18105 12.38 MB/sec warmup 47 sec latency 61.909 ms 2024-09-17T11:56:47.671 INFO:teuthology.orchestra.run.smithi045.stdout: 5 18334 12.16 MB/sec warmup 48 sec latency 79.048 ms 2024-09-17T11:56:48.022 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:47 smithi045 ceph-mon[23737]: pgmap v265: 97 pgs: 97 active+clean; 173 MiB data, 623 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.6 MiB/s wr, 632 op/s 2024-09-17T11:56:48.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:47 smithi165 ceph-mon[25702]: pgmap v265: 97 pgs: 97 active+clean; 173 MiB data, 623 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.6 MiB/s wr, 632 op/s 2024-09-17T11:56:48.671 INFO:teuthology.orchestra.run.smithi045.stdout: 5 18787 12.03 MB/sec warmup 49 sec latency 64.852 ms 2024-09-17T11:56:49.671 INFO:teuthology.orchestra.run.smithi045.stdout: 5 19402 12.13 MB/sec warmup 50 sec latency 54.046 ms 2024-09-17T11:56:50.041 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:49 smithi045 ceph-mon[23737]: pgmap v266: 97 pgs: 97 active+clean; 182 MiB data, 627 MiB used, 715 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.6 MiB/s wr, 599 op/s 2024-09-17T11:56:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:49 smithi165 ceph-mon[25702]: pgmap v266: 97 pgs: 97 active+clean; 182 MiB data, 627 MiB used, 715 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.6 MiB/s wr, 599 op/s 2024-09-17T11:56:50.671 INFO:teuthology.orchestra.run.smithi045.stdout: 5 20123 12.10 MB/sec warmup 51 sec latency 34.855 ms 2024-09-17T11:56:51.049 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:50 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:56:51.049 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:50 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:56:51.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:50 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:56:51.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:50 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:56:51.671 INFO:teuthology.orchestra.run.smithi045.stdout: 5 20651 12.30 MB/sec warmup 52 sec latency 65.106 ms 2024-09-17T11:56:52.061 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:51 smithi045 ceph-mon[23737]: pgmap v267: 97 pgs: 97 active+clean; 180 MiB data, 668 MiB used, 715 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.4 MiB/s wr, 488 op/s 2024-09-17T11:56:52.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:51 smithi165 ceph-mon[25702]: pgmap v267: 97 pgs: 97 active+clean; 180 MiB data, 668 MiB used, 715 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.4 MiB/s wr, 488 op/s 2024-09-17T11:56:52.671 INFO:teuthology.orchestra.run.smithi045.stdout: 5 21171 12.45 MB/sec warmup 53 sec latency 57.119 ms 2024-09-17T11:56:53.678 INFO:teuthology.orchestra.run.smithi045.stdout: 5 21494 12.47 MB/sec warmup 54 sec latency 72.124 ms 2024-09-17T11:56:53.914 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:53 smithi045 ceph-mon[23737]: pgmap v268: 97 pgs: 97 active+clean; 181 MiB data, 677 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 5.1 MiB/s wr, 528 op/s 2024-09-17T11:56:54.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:53 smithi165 ceph-mon[25702]: pgmap v268: 97 pgs: 97 active+clean; 181 MiB data, 677 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 5.1 MiB/s wr, 528 op/s 2024-09-17T11:56:54.678 INFO:teuthology.orchestra.run.smithi045.stdout: 5 21717 12.32 MB/sec warmup 55 sec latency 55.545 ms 2024-09-17T11:56:55.678 INFO:teuthology.orchestra.run.smithi045.stdout: 5 21939 12.14 MB/sec warmup 56 sec latency 64.786 ms 2024-09-17T11:56:55.970 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:55 smithi045 ceph-mon[23737]: pgmap v269: 97 pgs: 97 active+clean; 186 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.7 MiB/s wr, 661 op/s 2024-09-17T11:56:56.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:55 smithi165 ceph-mon[25702]: pgmap v269: 97 pgs: 97 active+clean; 186 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.7 MiB/s wr, 661 op/s 2024-09-17T11:56:56.678 INFO:teuthology.orchestra.run.smithi045.stdout: 5 22278 12.00 MB/sec warmup 57 sec latency 90.579 ms 2024-09-17T11:56:57.255 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T11:56:57.678 INFO:teuthology.orchestra.run.smithi045.stdout: 5 22607 11.96 MB/sec warmup 58 sec latency 69.241 ms 2024-09-17T11:56:57.998 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:57 smithi045 ceph-mon[23737]: pgmap v270: 97 pgs: 97 active+clean; 192 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.8 MiB/s wr, 605 op/s 2024-09-17T11:56:57.999 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:57 smithi045 ceph-mon[23737]: from='client.24413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:56:58.067 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T11:56:58.067 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (6m) 61s ago 7m 22.2M - ba2b418f427c c97407c70fb6 2024-09-17T11:56:58.068 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (7m) 61s ago 7m 6413k - 17.2.0 e1d6a67b021e 8c510943b55e 2024-09-17T11:56:58.068 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (6m) 63s ago 6m 6409k - 17.2.0 e1d6a67b021e 687c24a87b70 2024-09-17T11:56:58.068 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (5m) 61s ago 7m 53.5M - 8.3.5 dad864ee21e9 7d87d4101e56 2024-09-17T11:56:58.068 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (92s) 61s ago 91s 13.7M - 17.2.0 e1d6a67b021e 6a2a4a998dd1 2024-09-17T11:56:58.068 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (89s) 63s ago 89s 13.3M - 17.2.0 e1d6a67b021e d4c2987b9bac 2024-09-17T11:56:58.068 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:9283 running (8m) 61s ago 8m 460M - 17.2.0 e1d6a67b021e 5f496de39b08 2024-09-17T11:56:58.068 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283 running (6m) 63s ago 6m 411M - 17.2.0 e1d6a67b021e fefbdcc12a47 2024-09-17T11:56:58.068 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (8m) 61s ago 8m 47.9M 2048M 17.2.0 e1d6a67b021e c4256bdf51b7 2024-09-17T11:56:58.068 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (6m) 63s ago 6m 34.7M 2048M 17.2.0 e1d6a67b021e 9219263d3aa8 2024-09-17T11:56:58.068 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (71s) 61s ago 71s 49.0M - 4.0 e1d6a67b021e b8fc749ebb71 2024-09-17T11:56:58.068 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (66s) 63s ago 65s 48.8M - 4.0 e1d6a67b021e 771b27764432 2024-09-17T11:56:58.068 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (7m) 61s ago 7m 17.5M - 1dbe0e931976 09dea232cb6b 2024-09-17T11:56:58.068 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (6m) 63s ago 6m 16.6M - 1dbe0e931976 fdb9449adb13 2024-09-17T11:56:58.069 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (6m) 61s ago 6m 42.5M 3217M 17.2.0 e1d6a67b021e e0ce4d29dc64 2024-09-17T11:56:58.069 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (5m) 61s ago 5m 43.7M 3217M 17.2.0 e1d6a67b021e da8bda7b47b3 2024-09-17T11:56:58.069 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (5m) 61s ago 5m 40.4M 3217M 17.2.0 e1d6a67b021e 21fe0d9e5ced 2024-09-17T11:56:58.069 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (4m) 61s ago 4m 45.3M 3217M 17.2.0 e1d6a67b021e 92a2389488e4 2024-09-17T11:56:58.069 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (4m) 63s ago 4m 42.1M 3984M 17.2.0 e1d6a67b021e e29eb74df25a 2024-09-17T11:56:58.069 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (3m) 63s ago 3m 40.4M 3984M 17.2.0 e1d6a67b021e 5d900eb9165b 2024-09-17T11:56:58.069 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (3m) 63s ago 3m 42.0M 3984M 17.2.0 e1d6a67b021e fae3e1bae466 2024-09-17T11:56:58.069 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (3m) 63s ago 3m 39.6M 3984M 17.2.0 e1d6a67b021e 71b9f2cdb810 2024-09-17T11:56:58.069 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (6m) 61s ago 6m 60.0M - 514e6a882f6e 528c529e18d6 2024-09-17T11:56:58.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:57 smithi165 ceph-mon[25702]: pgmap v270: 97 pgs: 97 active+clean; 192 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.8 MiB/s wr, 605 op/s 2024-09-17T11:56:58.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:57 smithi165 ceph-mon[25702]: from='client.24413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:56:58.616 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T11:56:58.616 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T11:56:58.617 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T11:56:58.617 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:56:58.617 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T11:56:58.617 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T11:56:58.617 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:56:58.617 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T11:56:58.617 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T11:56:58.617 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:56:58.617 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T11:56:58.617 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T11:56:58.617 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:56:58.617 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T11:56:58.617 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-17T11:56:58.618 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T11:56:58.618 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T11:56:58.678 INFO:teuthology.orchestra.run.smithi045.stdout: 5 22943 11.90 MB/sec warmup 59 sec latency 81.689 ms 2024-09-17T11:56:58.901 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:58 smithi045 ceph-mon[23737]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:56:58.901 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:58 smithi045 ceph-mon[23737]: from='client.14638 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:56:58.901 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:58 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/1400050188' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T11:56:59.132 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T11:56:59.133 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:7498493279982dfe87f00616198a5967475f6169", 2024-09-17T11:56:59.133 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T11:56:59.133 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [], 2024-09-17T11:56:59.133 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "", 2024-09-17T11:56:59.133 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:7498493279982dfe87f00616198a5967475f6169 image" 2024-09-17T11:56:59.133 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T11:56:59.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:58 smithi165 ceph-mon[25702]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:56:59.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:58 smithi165 ceph-mon[25702]: from='client.14638 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:56:59.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:58 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/1400050188' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T11:56:59.651 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_OK 2024-09-17T11:56:59.678 INFO:teuthology.orchestra.run.smithi045.stdout: 5 23412 11.81 MB/sec warmup 60 sec latency 77.124 ms 2024-09-17T11:57:00.049 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:59 smithi045 ceph-mon[23737]: pgmap v271: 97 pgs: 97 active+clean; 189 MiB data, 782 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 553 op/s 2024-09-17T11:57:00.049 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:59 smithi045 ceph-mon[23737]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:57:00.049 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:56:59 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/1900597754' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T11:57:00.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:59 smithi165 ceph-mon[25702]: pgmap v271: 97 pgs: 97 active+clean; 189 MiB data, 782 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.0 MiB/s wr, 553 op/s 2024-09-17T11:57:00.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:59 smithi165 ceph-mon[25702]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:57:00.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:56:59 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/1900597754' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T11:57:00.678 INFO:teuthology.orchestra.run.smithi045.stdout: 5 23832 11.78 MB/sec warmup 61 sec latency 43.313 ms 2024-09-17T11:57:01.678 INFO:teuthology.orchestra.run.smithi045.stdout: 5 24217 11.85 MB/sec warmup 62 sec latency 41.629 ms 2024-09-17T11:57:02.079 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:01 smithi045 ceph-mon[23737]: pgmap v272: 97 pgs: 97 active+clean; 190 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.6 MiB/s wr, 567 op/s 2024-09-17T11:57:02.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:01 smithi165 ceph-mon[25702]: pgmap v272: 97 pgs: 97 active+clean; 190 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.6 MiB/s wr, 567 op/s 2024-09-17T11:57:02.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 24692 11.99 MB/sec warmup 63 sec latency 75.489 ms 2024-09-17T11:57:03.678 INFO:teuthology.orchestra.run.smithi045.stdout: 5 25148 12.07 MB/sec warmup 64 sec latency 40.415 ms 2024-09-17T11:57:03.845 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:03 smithi045 ceph-mon[23737]: pgmap v273: 97 pgs: 97 active+clean; 189 MiB data, 794 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.2 MiB/s wr, 530 op/s 2024-09-17T11:57:04.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:03 smithi165 ceph-mon[25702]: pgmap v273: 97 pgs: 97 active+clean; 189 MiB data, 794 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.2 MiB/s wr, 530 op/s 2024-09-17T11:57:04.678 INFO:teuthology.orchestra.run.smithi045.stdout: 5 25569 11.93 MB/sec warmup 65 sec latency 29.727 ms 2024-09-17T11:57:05.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 26025 11.91 MB/sec warmup 66 sec latency 64.764 ms 2024-09-17T11:57:05.877 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:05 smithi045 ceph-mon[23737]: pgmap v274: 97 pgs: 97 active+clean; 203 MiB data, 809 MiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.7 MiB/s wr, 670 op/s 2024-09-17T11:57:06.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:05 smithi165 ceph-mon[25702]: pgmap v274: 97 pgs: 97 active+clean; 203 MiB data, 809 MiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.7 MiB/s wr, 670 op/s 2024-09-17T11:57:06.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 26678 11.96 MB/sec warmup 67 sec latency 24.482 ms 2024-09-17T11:57:07.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 27275 11.91 MB/sec warmup 68 sec latency 32.408 ms 2024-09-17T11:57:07.903 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:07 smithi045 ceph-mon[23737]: pgmap v275: 97 pgs: 97 active+clean; 211 MiB data, 861 MiB used, 714 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.3 MiB/s wr, 618 op/s 2024-09-17T11:57:08.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:07 smithi165 ceph-mon[25702]: pgmap v275: 97 pgs: 97 active+clean; 211 MiB data, 861 MiB used, 714 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.3 MiB/s wr, 618 op/s 2024-09-17T11:57:08.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 27857 12.12 MB/sec warmup 69 sec latency 35.099 ms 2024-09-17T11:57:09.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 28275 12.14 MB/sec warmup 70 sec latency 70.862 ms 2024-09-17T11:57:09.938 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:09 smithi045 ceph-mon[23737]: pgmap v276: 97 pgs: 97 active+clean; 207 MiB data, 904 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.9 MiB/s wr, 585 op/s 2024-09-17T11:57:10.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:09 smithi165 ceph-mon[25702]: pgmap v276: 97 pgs: 97 active+clean; 207 MiB data, 904 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.9 MiB/s wr, 585 op/s 2024-09-17T11:57:10.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 28653 12.22 MB/sec warmup 71 sec latency 44.253 ms 2024-09-17T11:57:11.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 28896 12.09 MB/sec warmup 72 sec latency 53.522 ms 2024-09-17T11:57:11.973 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:11 smithi045 ceph-mon[23737]: pgmap v277: 97 pgs: 97 active+clean; 209 MiB data, 910 MiB used, 714 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.5 MiB/s wr, 661 op/s 2024-09-17T11:57:12.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:11 smithi165 ceph-mon[25702]: pgmap v277: 97 pgs: 97 active+clean; 209 MiB data, 910 MiB used, 714 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.5 MiB/s wr, 661 op/s 2024-09-17T11:57:12.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 29362 11.98 MB/sec warmup 73 sec latency 32.602 ms 2024-09-17T11:57:13.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 29813 11.98 MB/sec warmup 74 sec latency 84.022 ms 2024-09-17T11:57:14.027 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:13 smithi045 ceph-mon[23737]: pgmap v278: 97 pgs: 97 active+clean; 213 MiB data, 927 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 644 op/s 2024-09-17T11:57:14.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:13 smithi165 ceph-mon[25702]: pgmap v278: 97 pgs: 97 active+clean; 213 MiB data, 927 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 644 op/s 2024-09-17T11:57:14.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 30550 11.99 MB/sec warmup 75 sec latency 33.068 ms 2024-09-17T11:57:15.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 31093 12.06 MB/sec warmup 76 sec latency 52.889 ms 2024-09-17T11:57:15.944 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:15 smithi165 ceph-mon[25702]: pgmap v279: 97 pgs: 97 active+clean; 218 MiB data, 934 MiB used, 714 GiB / 715 GiB avail; 7.4 KiB/s rd, 7.4 MiB/s wr, 667 op/s 2024-09-17T11:57:15.944 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:15 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:57:15.944 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:15 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:57:15.944 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:15 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:57:16.087 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:15 smithi045 ceph-mon[23737]: pgmap v279: 97 pgs: 97 active+clean; 218 MiB data, 934 MiB used, 714 GiB / 715 GiB avail; 7.4 KiB/s rd, 7.4 MiB/s wr, 667 op/s 2024-09-17T11:57:16.087 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:15 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:57:16.087 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:15 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:57:16.087 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:15 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:57:16.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 31482 12.15 MB/sec warmup 77 sec latency 64.108 ms 2024-09-17T11:57:17.103 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:16 smithi045 ceph-mon[23737]: Upgrade: Target is version 19.3.0-4937-g74984932 (unknown) 2024-09-17T11:57:17.103 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:16 smithi045 ceph-mon[23737]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb'] 2024-09-17T11:57:17.103 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:16 smithi045 ceph-mon[23737]: Upgrade: Need to upgrade myself (mgr.smithi045.jtwyrp) 2024-09-17T11:57:17.103 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:16 smithi045 ceph-mon[23737]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb on smithi165 2024-09-17T11:57:17.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:16 smithi165 ceph-mon[25702]: Upgrade: Target is version 19.3.0-4937-g74984932 (unknown) 2024-09-17T11:57:17.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:16 smithi165 ceph-mon[25702]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, digests ['quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb'] 2024-09-17T11:57:17.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:16 smithi165 ceph-mon[25702]: Upgrade: Need to upgrade myself (mgr.smithi045.jtwyrp) 2024-09-17T11:57:17.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:16 smithi165 ceph-mon[25702]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb on smithi165 2024-09-17T11:57:17.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 31824 12.13 MB/sec warmup 78 sec latency 48.345 ms 2024-09-17T11:57:18.116 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:17 smithi045 ceph-mon[23737]: pgmap v280: 97 pgs: 97 active+clean; 219 MiB data, 965 MiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.6 MiB/s wr, 575 op/s 2024-09-17T11:57:18.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:17 smithi165 ceph-mon[25702]: pgmap v280: 97 pgs: 97 active+clean; 219 MiB data, 965 MiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.6 MiB/s wr, 575 op/s 2024-09-17T11:57:18.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 32178 12.16 MB/sec warmup 79 sec latency 63.956 ms 2024-09-17T11:57:19.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 32569 12.09 MB/sec warmup 80 sec latency 62.317 ms 2024-09-17T11:57:19.944 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:19 smithi045 ceph-mon[23737]: pgmap v281: 97 pgs: 97 active+clean; 222 MiB data, 993 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.8 MiB/s wr, 628 op/s 2024-09-17T11:57:20.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:19 smithi165 ceph-mon[25702]: pgmap v281: 97 pgs: 97 active+clean; 222 MiB data, 993 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.8 MiB/s wr, 628 op/s 2024-09-17T11:57:20.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 33233 12.08 MB/sec warmup 81 sec latency 20.970 ms 2024-09-17T11:57:21.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 33873 12.12 MB/sec warmup 82 sec latency 49.894 ms 2024-09-17T11:57:21.964 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:21 smithi045 ceph-mon[23737]: pgmap v282: 97 pgs: 97 active+clean; 229 MiB data, 1016 MiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 7.0 MiB/s wr, 682 op/s 2024-09-17T11:57:22.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:21 smithi165 ceph-mon[25702]: pgmap v282: 97 pgs: 97 active+clean; 229 MiB data, 1016 MiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 7.0 MiB/s wr, 682 op/s 2024-09-17T11:57:22.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 34437 12.12 MB/sec warmup 83 sec latency 46.260 ms 2024-09-17T11:57:23.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 35013 12.27 MB/sec warmup 84 sec latency 43.566 ms 2024-09-17T11:57:23.988 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:23 smithi045 ceph-mon[23737]: pgmap v283: 97 pgs: 97 active+clean; 237 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.7 MiB/s wr, 665 op/s 2024-09-17T11:57:24.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:23 smithi165 ceph-mon[25702]: pgmap v283: 97 pgs: 97 active+clean; 237 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.7 MiB/s wr, 665 op/s 2024-09-17T11:57:24.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 35740 12.44 MB/sec warmup 85 sec latency 50.415 ms 2024-09-17T11:57:25.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 36358 12.38 MB/sec warmup 86 sec latency 40.853 ms 2024-09-17T11:57:25.997 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:25 smithi045 ceph-mon[23737]: pgmap v284: 97 pgs: 97 active+clean; 230 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.8 MiB/s wr, 623 op/s 2024-09-17T11:57:26.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:25 smithi165 ceph-mon[25702]: pgmap v284: 97 pgs: 97 active+clean; 230 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.8 MiB/s wr, 623 op/s 2024-09-17T11:57:26.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 37069 12.46 MB/sec warmup 87 sec latency 22.396 ms 2024-09-17T11:57:27.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 37852 12.46 MB/sec warmup 88 sec latency 32.942 ms 2024-09-17T11:57:28.005 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:27 smithi045 ceph-mon[23737]: pgmap v285: 97 pgs: 97 active+clean; 233 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.3 MiB/s wr, 711 op/s 2024-09-17T11:57:28.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:27 smithi165 ceph-mon[25702]: pgmap v285: 97 pgs: 97 active+clean; 233 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.3 MiB/s wr, 711 op/s 2024-09-17T11:57:28.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 38528 12.65 MB/sec warmup 89 sec latency 21.835 ms 2024-09-17T11:57:29.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 39318 12.82 MB/sec warmup 90 sec latency 19.526 ms 2024-09-17T11:57:30.019 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:29 smithi045 ceph-mon[23737]: pgmap v286: 97 pgs: 97 active+clean; 239 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.2 MiB/s wr, 775 op/s 2024-09-17T11:57:30.243 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T11:57:30.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:29 smithi165 ceph-mon[25702]: pgmap v286: 97 pgs: 97 active+clean; 239 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.2 MiB/s wr, 775 op/s 2024-09-17T11:57:30.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 39868 12.74 MB/sec warmup 91 sec latency 19.240 ms 2024-09-17T11:57:31.024 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:30 smithi045 ceph-mon[23737]: from='client.24433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:57:31.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:30 smithi165 ceph-mon[25702]: from='client.24433 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:57:31.302 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T11:57:31.302 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (6m) 94s ago 8m 22.2M - ba2b418f427c c97407c70fb6 2024-09-17T11:57:31.302 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (8m) 94s ago 8m 6413k - 17.2.0 e1d6a67b021e 8c510943b55e 2024-09-17T11:57:31.302 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (7m) 96s ago 7m 6409k - 17.2.0 e1d6a67b021e 687c24a87b70 2024-09-17T11:57:31.302 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (6m) 94s ago 7m 53.5M - 8.3.5 dad864ee21e9 7d87d4101e56 2024-09-17T11:57:31.302 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (2m) 94s ago 2m 13.7M - 17.2.0 e1d6a67b021e 6a2a4a998dd1 2024-09-17T11:57:31.302 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (2m) 96s ago 2m 13.3M - 17.2.0 e1d6a67b021e d4c2987b9bac 2024-09-17T11:57:31.302 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:9283 running (9m) 94s ago 9m 460M - 17.2.0 e1d6a67b021e 5f496de39b08 2024-09-17T11:57:31.302 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283 running (7m) 96s ago 7m 411M - 17.2.0 e1d6a67b021e fefbdcc12a47 2024-09-17T11:57:31.303 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (9m) 94s ago 9m 47.9M 2048M 17.2.0 e1d6a67b021e c4256bdf51b7 2024-09-17T11:57:31.303 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (7m) 96s ago 7m 34.7M 2048M 17.2.0 e1d6a67b021e 9219263d3aa8 2024-09-17T11:57:31.303 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (104s) 94s ago 104s 49.0M - 4.0 e1d6a67b021e b8fc749ebb71 2024-09-17T11:57:31.303 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (99s) 96s ago 99s 48.8M - 4.0 e1d6a67b021e 771b27764432 2024-09-17T11:57:31.303 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (7m) 94s ago 7m 17.5M - 1dbe0e931976 09dea232cb6b 2024-09-17T11:57:31.303 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (7m) 96s ago 7m 16.6M - 1dbe0e931976 fdb9449adb13 2024-09-17T11:57:31.303 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (6m) 94s ago 6m 42.5M 3217M 17.2.0 e1d6a67b021e e0ce4d29dc64 2024-09-17T11:57:31.303 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (6m) 94s ago 6m 43.7M 3217M 17.2.0 e1d6a67b021e da8bda7b47b3 2024-09-17T11:57:31.303 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (5m) 94s ago 5m 40.4M 3217M 17.2.0 e1d6a67b021e 21fe0d9e5ced 2024-09-17T11:57:31.303 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (5m) 94s ago 5m 45.3M 3217M 17.2.0 e1d6a67b021e 92a2389488e4 2024-09-17T11:57:31.303 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (4m) 96s ago 4m 42.1M 3984M 17.2.0 e1d6a67b021e e29eb74df25a 2024-09-17T11:57:31.303 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (4m) 96s ago 4m 40.4M 3984M 17.2.0 e1d6a67b021e 5d900eb9165b 2024-09-17T11:57:31.303 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (4m) 96s ago 4m 42.0M 3984M 17.2.0 e1d6a67b021e fae3e1bae466 2024-09-17T11:57:31.304 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (3m) 96s ago 3m 39.6M 3984M 17.2.0 e1d6a67b021e 71b9f2cdb810 2024-09-17T11:57:31.304 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (7m) 94s ago 7m 60.0M - 514e6a882f6e 528c529e18d6 2024-09-17T11:57:31.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 40622 12.82 MB/sec warmup 92 sec latency 32.827 ms 2024-09-17T11:57:31.947 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T11:57:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T11:57:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T11:57:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:57:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T11:57:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T11:57:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:57:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T11:57:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T11:57:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:57:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T11:57:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T11:57:31.948 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:57:31.949 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T11:57:31.949 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-17T11:57:31.949 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T11:57:31.949 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T11:57:32.029 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:31 smithi045 ceph-mon[23737]: pgmap v287: 97 pgs: 97 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.6 KiB/s rd, 8.1 MiB/s wr, 785 op/s 2024-09-17T11:57:32.029 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:31 smithi045 ceph-mon[23737]: from='client.24437 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:57:32.029 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:31 smithi045 ceph-mon[23737]: from='client.24441 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:57:32.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:31 smithi165 ceph-mon[25702]: pgmap v287: 97 pgs: 97 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.6 KiB/s rd, 8.1 MiB/s wr, 785 op/s 2024-09-17T11:57:32.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:31 smithi165 ceph-mon[25702]: from='client.24437 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:57:32.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:31 smithi165 ceph-mon[25702]: from='client.24441 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:57:32.486 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T11:57:32.486 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T11:57:32.486 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T11:57:32.486 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [], 2024-09-17T11:57:32.486 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "0/23 daemons upgraded", 2024-09-17T11:57:32.486 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb image on host smithi165" 2024-09-17T11:57:32.486 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T11:57:32.679 INFO:teuthology.orchestra.run.smithi045.stdout: 5 41334 12.80 MB/sec warmup 93 sec latency 30.283 ms 2024-09-17T11:57:33.035 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:32 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/4186805878' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T11:57:33.133 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_OK 2024-09-17T11:57:33.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:32 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/4186805878' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T11:57:33.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 41983 12.94 MB/sec warmup 94 sec latency 18.274 ms 2024-09-17T11:57:34.039 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:33 smithi045 ceph-mon[23737]: from='client.24449 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:57:34.039 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:33 smithi045 ceph-mon[23737]: pgmap v288: 97 pgs: 97 active+clean; 251 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.3 MiB/s wr, 815 op/s 2024-09-17T11:57:34.039 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:33 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/17055538' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T11:57:34.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:33 smithi165 ceph-mon[25702]: from='client.24449 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:57:34.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:33 smithi165 ceph-mon[25702]: pgmap v288: 97 pgs: 97 active+clean; 251 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.3 MiB/s wr, 815 op/s 2024-09-17T11:57:34.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:33 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/17055538' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T11:57:34.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 42782 13.13 MB/sec warmup 95 sec latency 27.016 ms 2024-09-17T11:57:35.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 43036 13.04 MB/sec warmup 96 sec latency 73.146 ms 2024-09-17T11:57:36.086 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:35 smithi045 ceph-mon[23737]: pgmap v289: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 9.2 MiB/s wr, 813 op/s 2024-09-17T11:57:36.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:35 smithi165 ceph-mon[25702]: pgmap v289: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 9.2 MiB/s wr, 813 op/s 2024-09-17T11:57:36.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 43334 12.94 MB/sec warmup 97 sec latency 82.816 ms 2024-09-17T11:57:37.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 43606 12.86 MB/sec warmup 98 sec latency 73.698 ms 2024-09-17T11:57:38.182 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:37 smithi045 ceph-mon[23737]: pgmap v290: 97 pgs: 97 active+clean; 260 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.8 KiB/s rd, 10 MiB/s wr, 947 op/s 2024-09-17T11:57:38.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:37 smithi165 ceph-mon[25702]: pgmap v290: 97 pgs: 97 active+clean; 260 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.8 KiB/s rd, 10 MiB/s wr, 947 op/s 2024-09-17T11:57:38.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 43871 12.78 MB/sec warmup 99 sec latency 89.799 ms 2024-09-17T11:57:39.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 44204 12.76 MB/sec warmup 100 sec latency 71.638 ms 2024-09-17T11:57:39.978 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:39 smithi045 ceph-mon[23737]: pgmap v291: 97 pgs: 97 active+clean; 271 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 983 op/s 2024-09-17T11:57:40.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:39 smithi165 ceph-mon[25702]: pgmap v291: 97 pgs: 97 active+clean; 271 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 10 MiB/s wr, 983 op/s 2024-09-17T11:57:40.681 INFO:teuthology.orchestra.run.smithi045.stdout: 5 44475 12.68 MB/sec warmup 101 sec latency 100.619 ms 2024-09-17T11:57:41.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 44851 12.58 MB/sec warmup 102 sec latency 74.710 ms 2024-09-17T11:57:42.022 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:41 smithi045 ceph-mon[23737]: pgmap v292: 97 pgs: 97 active+clean; 267 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.6 MiB/s wr, 808 op/s 2024-09-17T11:57:42.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:41 smithi165 ceph-mon[25702]: pgmap v292: 97 pgs: 97 active+clean; 267 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.6 MiB/s wr, 808 op/s 2024-09-17T11:57:42.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 45118 12.55 MB/sec warmup 103 sec latency 49.293 ms 2024-09-17T11:57:43.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 45615 12.67 MB/sec warmup 104 sec latency 65.933 ms 2024-09-17T11:57:44.069 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:43 smithi045 ceph-mon[23737]: pgmap v293: 97 pgs: 97 active+clean; 263 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.2 MiB/s wr, 660 op/s 2024-09-17T11:57:44.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:43 smithi165 ceph-mon[25702]: pgmap v293: 97 pgs: 97 active+clean; 263 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 7.2 MiB/s wr, 660 op/s 2024-09-17T11:57:44.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 46015 12.67 MB/sec warmup 105 sec latency 57.018 ms 2024-09-17T11:57:45.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 46374 12.68 MB/sec warmup 106 sec latency 57.221 ms 2024-09-17T11:57:46.132 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:45 smithi045 ceph-mon[23737]: pgmap v294: 97 pgs: 97 active+clean; 267 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.2 MiB/s wr, 677 op/s 2024-09-17T11:57:46.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:45 smithi165 ceph-mon[25702]: pgmap v294: 97 pgs: 97 active+clean; 267 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.2 MiB/s wr, 677 op/s 2024-09-17T11:57:46.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 46711 12.60 MB/sec warmup 107 sec latency 69.141 ms 2024-09-17T11:57:47.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 47247 12.58 MB/sec warmup 108 sec latency 34.280 ms 2024-09-17T11:57:48.160 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:47 smithi045 ceph-mon[23737]: pgmap v295: 97 pgs: 97 active+clean; 272 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.3 MiB/s wr, 643 op/s 2024-09-17T11:57:48.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:47 smithi165 ceph-mon[25702]: pgmap v295: 97 pgs: 97 active+clean; 272 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.3 MiB/s wr, 643 op/s 2024-09-17T11:57:48.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 47769 12.58 MB/sec warmup 109 sec latency 53.248 ms 2024-09-17T11:57:49.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 48334 12.53 MB/sec warmup 110 sec latency 44.550 ms 2024-09-17T11:57:49.954 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:49 smithi045 ceph-mon[23737]: pgmap v296: 97 pgs: 97 active+clean; 282 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.7 MiB/s wr, 562 op/s 2024-09-17T11:57:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:49 smithi165 ceph-mon[25702]: pgmap v296: 97 pgs: 97 active+clean; 282 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.7 MiB/s wr, 562 op/s 2024-09-17T11:57:50.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 48541 12.46 MB/sec warmup 111 sec latency 69.688 ms 2024-09-17T11:57:51.020 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:50 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:57:51.021 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:50 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:57:51.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:50 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:57:51.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:50 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:57:51.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 48700 12.41 MB/sec warmup 112 sec latency 97.243 ms 2024-09-17T11:57:52.070 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:51 smithi045 ceph-mon[23737]: pgmap v297: 97 pgs: 97 active+clean; 281 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.9 MiB/s wr, 463 op/s 2024-09-17T11:57:52.070 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:57:52.070 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi165.ykdszc", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T11:57:52.070 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T11:57:52.070 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:51 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:57:52.189 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:51 smithi165 ceph-mon[25702]: pgmap v297: 97 pgs: 97 active+clean; 281 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.9 MiB/s wr, 463 op/s 2024-09-17T11:57:52.189 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:51 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:57:52.189 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:51 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi165.ykdszc", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T11:57:52.189 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:51 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T11:57:52.189 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:51 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:57:52.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 49144 12.48 MB/sec warmup 113 sec latency 83.741 ms 2024-09-17T11:57:53.085 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:52 smithi045 ceph-mon[23737]: Upgrade: Updating mgr.smithi165.ykdszc 2024-09-17T11:57:53.085 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:52 smithi045 ceph-mon[23737]: Deploying daemon mgr.smithi165.ykdszc on smithi165 2024-09-17T11:57:53.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:52 smithi165 ceph-mon[25702]: Upgrade: Updating mgr.smithi165.ykdszc 2024-09-17T11:57:53.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:52 smithi165 ceph-mon[25702]: Deploying daemon mgr.smithi165.ykdszc on smithi165 2024-09-17T11:57:53.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 49731 12.55 MB/sec warmup 114 sec latency 45.986 ms 2024-09-17T11:57:54.099 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:53 smithi045 ceph-mon[23737]: pgmap v298: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 5.2 MiB/s wr, 554 op/s 2024-09-17T11:57:54.200 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:53 smithi165 ceph-mon[25702]: pgmap v298: 97 pgs: 97 active+clean; 284 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 5.2 MiB/s wr, 554 op/s 2024-09-17T11:57:54.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 50114 12.55 MB/sec warmup 115 sec latency 33.269 ms 2024-09-17T11:57:55.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 50517 12.47 MB/sec warmup 116 sec latency 38.567 ms 2024-09-17T11:57:56.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:55 smithi165 ceph-mon[25702]: pgmap v299: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.9 KiB/s rd, 6.0 MiB/s wr, 585 op/s 2024-09-17T11:57:56.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:55 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:57:56.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:55 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:57:56.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:55 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:57:56.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:55 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:57:56.134 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:55 smithi045 ceph-mon[23737]: pgmap v299: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.9 KiB/s rd, 6.0 MiB/s wr, 585 op/s 2024-09-17T11:57:56.134 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:55 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:57:56.134 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:55 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:57:56.134 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:55 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:57:56.134 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:55 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:57:56.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 50880 12.43 MB/sec warmup 117 sec latency 50.773 ms 2024-09-17T11:57:57.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 51289 12.41 MB/sec warmup 118 sec latency 52.844 ms 2024-09-17T11:57:58.156 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:57 smithi045 ceph-mon[23737]: pgmap v300: 97 pgs: 97 active+clean; 291 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.2 MiB/s wr, 549 op/s 2024-09-17T11:57:58.248 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:57 smithi165 ceph-mon[25702]: pgmap v300: 97 pgs: 97 active+clean; 291 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.2 MiB/s wr, 549 op/s 2024-09-17T11:57:58.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 51776 12.38 MB/sec warmup 119 sec latency 35.817 ms 2024-09-17T11:58:00.191 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:57:59 smithi045 ceph-mon[23737]: pgmap v301: 97 pgs: 97 active+clean; 287 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 510 op/s 2024-09-17T11:58:00.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:57:59 smithi165 ceph-mon[25702]: pgmap v301: 97 pgs: 97 active+clean; 287 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 510 op/s 2024-09-17T11:58:00.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 52626 18.65 MB/sec execute 1 sec latency 81.112 ms 2024-09-17T11:58:01.202 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:01 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:58:01.202 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:01 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:58:01.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:01 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:58:01.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:01 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:58:01.681 INFO:teuthology.orchestra.run.smithi045.stdout: 5 53196 21.15 MB/sec execute 2 sec latency 39.750 ms 2024-09-17T11:58:02.229 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:02 smithi045 ceph-mon[23737]: pgmap v302: 97 pgs: 97 active+clean; 289 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.4 MiB/s wr, 499 op/s 2024-09-17T11:58:02.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:02 smithi165 ceph-mon[25702]: pgmap v302: 97 pgs: 97 active+clean; 289 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.4 MiB/s wr, 499 op/s 2024-09-17T11:58:02.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 53586 18.29 MB/sec execute 3 sec latency 45.445 ms 2024-09-17T11:58:03.092 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:03 smithi165 ceph-mon[25702]: Standby manager daemon smithi165.ykdszc restarted 2024-09-17T11:58:03.092 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:03 smithi165 ceph-mon[25702]: Standby manager daemon smithi165.ykdszc started 2024-09-17T11:58:03.092 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:03 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi165.ykdszc/crt"}]: dispatch 2024-09-17T11:58:03.092 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:03 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T11:58:03.092 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:03 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi165.ykdszc/key"}]: dispatch 2024-09-17T11:58:03.092 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:03 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T11:58:03.114 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:03 smithi045 ceph-mon[23737]: Standby manager daemon smithi165.ykdszc restarted 2024-09-17T11:58:03.114 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:03 smithi045 ceph-mon[23737]: Standby manager daemon smithi165.ykdszc started 2024-09-17T11:58:03.114 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:03 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi165.ykdszc/crt"}]: dispatch 2024-09-17T11:58:03.114 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:03 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T11:58:03.114 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:03 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi165.ykdszc/key"}]: dispatch 2024-09-17T11:58:03.114 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:03 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T11:58:03.602 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T11:58:03.680 INFO:teuthology.orchestra.run.smithi045.stdout: 5 53929 14.44 MB/sec execute 4 sec latency 47.737 ms 2024-09-17T11:58:04.247 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:04 smithi165 ceph-mon[25702]: pgmap v303: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.6 MiB/s wr, 590 op/s 2024-09-17T11:58:04.247 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:04 smithi165 ceph-mon[25702]: mgrmap e22: smithi045.jtwyrp(active, since 8m), standbys: smithi165.ykdszc 2024-09-17T11:58:04.252 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:04 smithi045 ceph-mon[23737]: pgmap v303: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 5.6 MiB/s wr, 590 op/s 2024-09-17T11:58:04.253 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:04 smithi045 ceph-mon[23737]: mgrmap e22: smithi045.jtwyrp(active, since 8m), standbys: smithi165.ykdszc 2024-09-17T11:58:04.436 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T11:58:04.436 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (7m) 2m ago 8m 22.2M - ba2b418f427c c97407c70fb6 2024-09-17T11:58:04.436 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (8m) 2m ago 8m 6413k - 17.2.0 e1d6a67b021e 8c510943b55e 2024-09-17T11:58:04.437 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (8m) 4s ago 8m 6409k - 17.2.0 e1d6a67b021e 687c24a87b70 2024-09-17T11:58:04.437 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (6m) 2m ago 8m 53.5M - 8.3.5 dad864ee21e9 7d87d4101e56 2024-09-17T11:58:04.437 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (2m) 2m ago 2m 13.7M - 17.2.0 e1d6a67b021e 6a2a4a998dd1 2024-09-17T11:58:04.437 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (2m) 4s ago 2m 13.9M - 17.2.0 e1d6a67b021e d4c2987b9bac 2024-09-17T11:58:04.437 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:9283 running (9m) 2m ago 9m 460M - 17.2.0 e1d6a67b021e 5f496de39b08 2024-09-17T11:58:04.437 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283 running (9s) 4s ago 7m 63.5M - 19.3.0-4937-g74984932 6c1fcab85602 96e7a3af0643 2024-09-17T11:58:04.437 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (9m) 2m ago 10m 47.9M 2048M 17.2.0 e1d6a67b021e c4256bdf51b7 2024-09-17T11:58:04.437 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (7m) 4s ago 7m 37.4M 2048M 17.2.0 e1d6a67b021e 9219263d3aa8 2024-09-17T11:58:04.437 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (2m) 2m ago 2m 49.0M - 4.0 e1d6a67b021e b8fc749ebb71 2024-09-17T11:58:04.437 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (2m) 4s ago 2m 75.0M - 4.0 e1d6a67b021e 771b27764432 2024-09-17T11:58:04.437 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (8m) 2m ago 8m 17.5M - 1dbe0e931976 09dea232cb6b 2024-09-17T11:58:04.437 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (7m) 4s ago 7m 20.3M - 1dbe0e931976 fdb9449adb13 2024-09-17T11:58:04.437 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (7m) 2m ago 7m 42.5M 3217M 17.2.0 e1d6a67b021e e0ce4d29dc64 2024-09-17T11:58:04.438 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (6m) 2m ago 6m 43.7M 3217M 17.2.0 e1d6a67b021e da8bda7b47b3 2024-09-17T11:58:04.438 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (6m) 2m ago 6m 40.4M 3217M 17.2.0 e1d6a67b021e 21fe0d9e5ced 2024-09-17T11:58:04.438 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (5m) 2m ago 5m 45.3M 3217M 17.2.0 e1d6a67b021e 92a2389488e4 2024-09-17T11:58:04.438 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (5m) 4s ago 5m 223M 3984M 17.2.0 e1d6a67b021e e29eb74df25a 2024-09-17T11:58:04.438 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (4m) 4s ago 4m 214M 3984M 17.2.0 e1d6a67b021e 5d900eb9165b 2024-09-17T11:58:04.438 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (4m) 4s ago 4m 201M 3984M 17.2.0 e1d6a67b021e fae3e1bae466 2024-09-17T11:58:04.438 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (4m) 4s ago 4m 308M 3984M 17.2.0 e1d6a67b021e 71b9f2cdb810 2024-09-17T11:58:04.438 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (7m) 2m ago 7m 60.0M - 514e6a882f6e 528c529e18d6 2024-09-17T11:58:04.681 INFO:teuthology.orchestra.run.smithi045.stdout: 5 54246 12.63 MB/sec execute 5 sec latency 61.250 ms 2024-09-17T11:58:04.981 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T11:58:04.981 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T11:58:04.982 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T11:58:04.982 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:58:04.982 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T11:58:04.982 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T11:58:04.982 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 1 2024-09-17T11:58:04.982 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:58:04.982 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T11:58:04.982 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T11:58:04.982 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:58:04.982 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T11:58:04.982 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T11:58:04.982 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:58:04.983 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T11:58:04.983 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-17T11:58:04.983 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 1 2024-09-17T11:58:04.983 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T11:58:04.983 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T11:58:05.282 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:05 smithi045 ceph-mon[23737]: from='client.14676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:58:05.282 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:05 smithi045 ceph-mon[23737]: from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:58:05.282 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:05 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:58:05.282 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:05 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:58:05.282 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:05 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:58:05.282 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:05 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:58:05.282 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:05 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mgr fail", "who": "smithi045.jtwyrp"}]: dispatch 2024-09-17T11:58:05.282 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:05 smithi045 ceph-mon[23737]: osdmap e70: 8 total, 8 up, 8 in 2024-09-17T11:58:05.282 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:05 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/123867510' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T11:58:05.372 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T11:58:05.372 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T11:58:05.372 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T11:58:05.372 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [], 2024-09-17T11:58:05.372 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "1/23 daemons upgraded", 2024-09-17T11:58:05.372 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "Currently upgrading mgr daemons" 2024-09-17T11:58:05.372 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T11:58:05.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:05 smithi165 ceph-mon[25702]: from='client.14676 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:58:05.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:05 smithi165 ceph-mon[25702]: from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:58:05.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:05 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:58:05.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:05 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:58:05.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:05 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:58:05.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:05 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:58:05.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:05 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mgr fail", "who": "smithi045.jtwyrp"}]: dispatch 2024-09-17T11:58:05.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:05 smithi165 ceph-mon[25702]: osdmap e70: 8 total, 8 up, 8 in 2024-09-17T11:58:05.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:05 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/123867510' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T11:58:05.681 INFO:teuthology.orchestra.run.smithi045.stdout: 5 54591 12.48 MB/sec execute 6 sec latency 42.421 ms 2024-09-17T11:58:05.919 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_OK 2024-09-17T11:58:06.111 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: from='client.14684 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:58:06.111 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: pgmap v304: 97 pgs: 97 active+clean; 299 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.2 MiB/s wr, 545 op/s 2024-09-17T11:58:06.111 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: Upgrade: Need to upgrade myself (mgr.smithi045.jtwyrp) 2024-09-17T11:58:06.111 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: Failing over to other MGR 2024-09-17T11:58:06.111 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "mon metadata", "id": "smithi045"}]: dispatch 2024-09-17T11:58:06.111 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "mon metadata", "id": "smithi165"}]: dispatch 2024-09-17T11:58:06.111 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi045.olwaas"}]: dispatch 2024-09-17T11:58:06.111 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi165.czpmgd"}]: dispatch 2024-09-17T11:58:06.111 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "mgr metadata", "who": "smithi165.ykdszc", "id": "smithi165.ykdszc"}]: dispatch 2024-09-17T11:58:06.111 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T11:58:06.111 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T11:58:06.111 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T11:58:06.112 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T11:58:06.112 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T11:58:06.112 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T11:58:06.112 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T11:58:06.112 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T11:58:06.112 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "mgr fail", "who": "smithi045.jtwyrp"}]': finished 2024-09-17T11:58:06.112 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: mgrmap e23: smithi165.ykdszc(active, starting, since 0.996365s) 2024-09-17T11:58:06.112 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T11:58:06.112 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T11:58:06.112 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T11:58:06.112 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/1528034957' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T11:58:06.112 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:06 smithi045 ceph-mon[23737]: Manager daemon smithi165.ykdszc is now available 2024-09-17T11:58:06.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: from='client.14684 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:58:06.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: pgmap v304: 97 pgs: 97 active+clean; 299 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.2 MiB/s wr, 545 op/s 2024-09-17T11:58:06.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: Upgrade: Need to upgrade myself (mgr.smithi045.jtwyrp) 2024-09-17T11:58:06.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: Failing over to other MGR 2024-09-17T11:58:06.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "mon metadata", "id": "smithi045"}]: dispatch 2024-09-17T11:58:06.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "mon metadata", "id": "smithi165"}]: dispatch 2024-09-17T11:58:06.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi045.olwaas"}]: dispatch 2024-09-17T11:58:06.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi165.czpmgd"}]: dispatch 2024-09-17T11:58:06.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "mgr metadata", "who": "smithi165.ykdszc", "id": "smithi165.ykdszc"}]: dispatch 2024-09-17T11:58:06.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T11:58:06.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T11:58:06.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T11:58:06.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T11:58:06.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T11:58:06.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T11:58:06.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T11:58:06.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T11:58:06.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: from='mgr.14200 172.21.15.45:0/2273563909' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "mgr fail", "who": "smithi045.jtwyrp"}]': finished 2024-09-17T11:58:06.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: mgrmap e23: smithi165.ykdszc(active, starting, since 0.996365s) 2024-09-17T11:58:06.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T11:58:06.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T11:58:06.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T11:58:06.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/1528034957' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T11:58:06.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:06 smithi165 ceph-mon[25702]: Manager daemon smithi165.ykdszc is now available 2024-09-17T11:58:06.681 INFO:teuthology.orchestra.run.smithi045.stdout: 5 54957 11.41 MB/sec execute 7 sec latency 58.565 ms 2024-09-17T11:58:07.681 INFO:teuthology.orchestra.run.smithi045.stdout: 5 55317 10.71 MB/sec execute 8 sec latency 63.962 ms 2024-09-17T11:58:07.918 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:07 smithi045 ceph-mon[23737]: mgrmap e24: smithi165.ykdszc(active, since 1.99551s) 2024-09-17T11:58:07.918 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:07 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:07.918 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:07 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:07.918 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:07 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:07.918 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:07 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:07.918 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:07 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:07.919 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:07 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:58:07.919 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:07 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T11:58:07.919 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:07 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T11:58:07.919 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:07 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi165.ykdszc/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:58:07.919 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:07 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi165.ykdszc/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:58:07.919 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:07 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi165.ykdszc/trash_purge_schedule"}]: dispatch 2024-09-17T11:58:07.919 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:07 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi165.ykdszc/trash_purge_schedule"}]: dispatch 2024-09-17T11:58:08.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:07 smithi165 ceph-mon[25702]: mgrmap e24: smithi165.ykdszc(active, since 1.99551s) 2024-09-17T11:58:08.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:07 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:08.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:07 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:08.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:07 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:08.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:07 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:08.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:07 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:08.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:07 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:58:08.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:07 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T11:58:08.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:07 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T11:58:08.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:07 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi165.ykdszc/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:58:08.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:07 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi165.ykdszc/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:58:08.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:07 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi165.ykdszc/trash_purge_schedule"}]: dispatch 2024-09-17T11:58:08.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:07 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi165.ykdszc/trash_purge_schedule"}]: dispatch 2024-09-17T11:58:08.681 INFO:teuthology.orchestra.run.smithi045.stdout: 5 55628 10.05 MB/sec execute 9 sec latency 80.881 ms 2024-09-17T11:58:08.950 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:08 smithi045 ceph-mon[23737]: Checking for cert/key for grafana.smithi045 2024-09-17T11:58:08.950 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:08 smithi045 ceph-mon[23737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T11:58:08.950 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:08 smithi045 ceph-mon[23737]: pgmap v4: 97 pgs: 97 active+clean; 298 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-17T11:58:09.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:08 smithi165 ceph-mon[25702]: Checking for cert/key for grafana.smithi045 2024-09-17T11:58:09.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:08 smithi165 ceph-mon[25702]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T11:58:09.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:08 smithi165 ceph-mon[25702]: pgmap v4: 97 pgs: 97 active+clean; 298 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-17T11:58:09.681 INFO:teuthology.orchestra.run.smithi045.stdout: 5 56011 10.90 MB/sec execute 10 sec latency 69.980 ms 2024-09-17T11:58:09.955 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:09 smithi045 ceph-mon[23737]: Deploying cephadm binary to smithi165 2024-09-17T11:58:09.955 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:09 smithi045 ceph-mon[23737]: Deploying cephadm binary to smithi045 2024-09-17T11:58:09.955 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:09 smithi045 ceph-mon[23737]: [17/Sep/2024:11:58:08] ENGINE Bus STARTING 2024-09-17T11:58:09.956 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:09 smithi045 ceph-mon[23737]: [17/Sep/2024:11:58:08] ENGINE Serving on https://172.21.15.165:7150 2024-09-17T11:58:09.956 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:09 smithi045 ceph-mon[23737]: [17/Sep/2024:11:58:08] ENGINE Error in HTTPServer.serve 2024-09-17T11:58:09.956 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Traceback (most recent call last): 2024-09-17T11:58:09.956 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T11:58:09.956 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: self._connections.run(self.expiration_interval) 2024-09-17T11:58:09.956 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T11:58:09.956 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: self._run(expiration_interval) 2024-09-17T11:58:09.956 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T11:58:09.956 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T11:58:09.956 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T11:58:09.956 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T11:58:09.957 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T11:58:09.957 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: s = self.context.wrap_socket( 2024-09-17T11:58:09.957 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T11:58:09.957 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: return self.sslsocket_class._create( 2024-09-17T11:58:09.957 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T11:58:09.957 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: self.do_handshake() 2024-09-17T11:58:09.957 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T11:58:09.957 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: self._sslobj.do_handshake() 2024-09-17T11:58:09.957 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T11:58:09.957 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:09 smithi045 ceph-mon[23737]: [17/Sep/2024:11:58:08] ENGINE Serving on http://172.21.15.165:8765 2024-09-17T11:58:09.958 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:09 smithi045 ceph-mon[23737]: [17/Sep/2024:11:58:08] ENGINE Bus STARTED 2024-09-17T11:58:09.958 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:09 smithi045 ceph-mon[23737]: mgrmap e25: smithi165.ykdszc(active, since 4s) 2024-09-17T11:58:10.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:09 smithi165 ceph-mon[25702]: Deploying cephadm binary to smithi165 2024-09-17T11:58:10.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:09 smithi165 ceph-mon[25702]: Deploying cephadm binary to smithi045 2024-09-17T11:58:10.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:09 smithi165 ceph-mon[25702]: [17/Sep/2024:11:58:08] ENGINE Bus STARTING 2024-09-17T11:58:10.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:09 smithi165 ceph-mon[25702]: [17/Sep/2024:11:58:08] ENGINE Serving on https://172.21.15.165:7150 2024-09-17T11:58:10.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:09 smithi165 ceph-mon[25702]: [17/Sep/2024:11:58:08] ENGINE Error in HTTPServer.serve 2024-09-17T11:58:10.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Traceback (most recent call last): 2024-09-17T11:58:10.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T11:58:10.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: self._connections.run(self.expiration_interval) 2024-09-17T11:58:10.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T11:58:10.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: self._run(expiration_interval) 2024-09-17T11:58:10.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T11:58:10.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T11:58:10.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T11:58:10.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T11:58:10.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T11:58:10.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: s = self.context.wrap_socket( 2024-09-17T11:58:10.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T11:58:10.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: return self.sslsocket_class._create( 2024-09-17T11:58:10.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T11:58:10.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: self.do_handshake() 2024-09-17T11:58:10.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T11:58:10.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: self._sslobj.do_handshake() 2024-09-17T11:58:10.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T11:58:10.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:09 smithi165 ceph-mon[25702]: [17/Sep/2024:11:58:08] ENGINE Serving on http://172.21.15.165:8765 2024-09-17T11:58:10.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:09 smithi165 ceph-mon[25702]: [17/Sep/2024:11:58:08] ENGINE Bus STARTED 2024-09-17T11:58:10.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:09 smithi165 ceph-mon[25702]: mgrmap e25: smithi165.ykdszc(active, since 4s) 2024-09-17T11:58:10.690 INFO:teuthology.orchestra.run.smithi045.stdout: 5 56639 12.24 MB/sec execute 11 sec latency 36.705 ms 2024-09-17T11:58:10.969 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:10 smithi045 ceph-mon[23737]: pgmap v5: 97 pgs: 97 active+clean; 298 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-17T11:58:11.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:10 smithi165 ceph-mon[25702]: pgmap v5: 97 pgs: 97 active+clean; 298 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-17T11:58:11.689 INFO:teuthology.orchestra.run.smithi045.stdout: 5 57007 12.33 MB/sec execute 12 sec latency 75.877 ms 2024-09-17T11:58:11.996 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:11 smithi045 ceph-mon[23737]: mgrmap e26: smithi165.ykdszc(active, since 6s) 2024-09-17T11:58:11.996 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:11 smithi045 ceph-mon[23737]: pgmap v6: 97 pgs: 97 active+clean; 302 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s wr, 99 op/s 2024-09-17T11:58:12.169 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:11 smithi165 ceph-mon[25702]: mgrmap e26: smithi165.ykdszc(active, since 6s) 2024-09-17T11:58:12.169 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:11 smithi165 ceph-mon[25702]: pgmap v6: 97 pgs: 97 active+clean; 302 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s wr, 99 op/s 2024-09-17T11:58:12.690 INFO:teuthology.orchestra.run.smithi045.stdout: 5 57286 11.67 MB/sec execute 13 sec latency 54.973 ms 2024-09-17T11:58:13.690 INFO:teuthology.orchestra.run.smithi045.stdout: 5 57602 11.09 MB/sec execute 14 sec latency 103.037 ms 2024-09-17T11:58:14.690 INFO:teuthology.orchestra.run.smithi045.stdout: 5 58000 11.10 MB/sec execute 15 sec latency 59.723 ms 2024-09-17T11:58:14.780 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:14 smithi045 ceph-mon[23737]: pgmap v7: 97 pgs: 97 active+clean; 316 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 4.8 MiB/s wr, 398 op/s 2024-09-17T11:58:14.780 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:14 smithi045 ceph-mon[23737]: Standby manager daemon smithi045.jtwyrp started 2024-09-17T11:58:14.780 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:14 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.45:0/3310731518' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi045.jtwyrp/crt"}]: dispatch 2024-09-17T11:58:14.780 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:14 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.45:0/3310731518' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T11:58:14.780 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:14 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.45:0/3310731518' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi045.jtwyrp/key"}]: dispatch 2024-09-17T11:58:14.780 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:14 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.45:0/3310731518' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T11:58:15.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:14 smithi165 ceph-mon[25702]: pgmap v7: 97 pgs: 97 active+clean; 316 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 4.8 MiB/s wr, 398 op/s 2024-09-17T11:58:15.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:14 smithi165 ceph-mon[25702]: Standby manager daemon smithi045.jtwyrp started 2024-09-17T11:58:15.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:14 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.45:0/3310731518' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi045.jtwyrp/crt"}]: dispatch 2024-09-17T11:58:15.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:14 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.45:0/3310731518' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T11:58:15.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:14 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.45:0/3310731518' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi045.jtwyrp/key"}]: dispatch 2024-09-17T11:58:15.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:14 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.45:0/3310731518' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T11:58:15.690 INFO:teuthology.orchestra.run.smithi045.stdout: 5 58381 10.92 MB/sec execute 16 sec latency 56.471 ms 2024-09-17T11:58:15.786 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:15 smithi045 ceph-mon[23737]: mgrmap e27: smithi165.ykdszc(active, since 10s), standbys: smithi045.jtwyrp 2024-09-17T11:58:15.787 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:15 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "mgr metadata", "who": "smithi045.jtwyrp", "id": "smithi045.jtwyrp"}]: dispatch 2024-09-17T11:58:16.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:15 smithi165 ceph-mon[25702]: mgrmap e27: smithi165.ykdszc(active, since 10s), standbys: smithi045.jtwyrp 2024-09-17T11:58:16.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:15 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "mgr metadata", "who": "smithi045.jtwyrp", "id": "smithi045.jtwyrp"}]: dispatch 2024-09-17T11:58:16.690 INFO:teuthology.orchestra.run.smithi045.stdout: 5 58919 10.74 MB/sec execute 17 sec latency 54.450 ms 2024-09-17T11:58:16.817 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:16 smithi045 ceph-mon[23737]: pgmap v8: 97 pgs: 97 active+clean; 316 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 3.7 MiB/s wr, 309 op/s 2024-09-17T11:58:17.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:16 smithi165 ceph-mon[25702]: pgmap v8: 97 pgs: 97 active+clean; 316 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 3.7 MiB/s wr, 309 op/s 2024-09-17T11:58:17.690 INFO:teuthology.orchestra.run.smithi045.stdout: 5 59310 10.68 MB/sec execute 18 sec latency 59.175 ms 2024-09-17T11:58:18.346 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:18 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:18.346 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:18 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:18.346 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:18 smithi045 ceph-mon[23737]: pgmap v9: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 5.2 MiB/s wr, 519 op/s 2024-09-17T11:58:18.346 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:18 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:18.346 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:18 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:18.346 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:18 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:18.346 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:18 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:18.346 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:18 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:18.347 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:18 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:18.347 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:18 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:18.347 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:18 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:18.347 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:18 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:18.347 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:18 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:18.347 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:18 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:18.609 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:18 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:18.609 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:18 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:18.609 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:18 smithi165 ceph-mon[25702]: pgmap v9: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 5.2 MiB/s wr, 519 op/s 2024-09-17T11:58:18.609 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:18 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:18.609 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:18 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:18.609 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:18 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:18.610 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:18 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:18.610 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:18 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:18.610 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:18 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:18.610 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:18 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:18.610 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:18 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:18.610 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:18 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:18.610 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:18 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:18.610 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:18 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:18.690 INFO:teuthology.orchestra.run.smithi045.stdout: 5 59775 11.14 MB/sec execute 19 sec latency 38.199 ms 2024-09-17T11:58:19.375 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:19 smithi045 ceph-mon[23737]: Adjusting osd_memory_target on smithi045 to 1937M 2024-09-17T11:58:19.643 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:19 smithi165 ceph-mon[25702]: Adjusting osd_memory_target on smithi045 to 1937M 2024-09-17T11:58:19.690 INFO:teuthology.orchestra.run.smithi045.stdout: 5 60275 11.66 MB/sec execute 20 sec latency 44.870 ms 2024-09-17T11:58:20.393 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:20 smithi045 ceph-mon[23737]: pgmap v10: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 4.7 MiB/s wr, 474 op/s 2024-09-17T11:58:20.393 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:20 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:20.393 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:20 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:20.690 INFO:teuthology.orchestra.run.smithi045.stdout: 5 60720 11.70 MB/sec execute 21 sec latency 33.285 ms 2024-09-17T11:58:20.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:20 smithi165 ceph-mon[25702]: pgmap v10: 97 pgs: 97 active+clean; 312 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 4.7 MiB/s wr, 474 op/s 2024-09-17T11:58:20.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:20 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:20.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:20 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:21.690 INFO:teuthology.orchestra.run.smithi045.stdout: 5 61097 11.39 MB/sec execute 22 sec latency 43.017 ms 2024-09-17T11:58:21.909 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:21 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:21.910 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:21 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:22.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:21 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:22.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:21 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:22.690 INFO:teuthology.orchestra.run.smithi045.stdout: 5 61503 11.26 MB/sec execute 23 sec latency 61.602 ms 2024-09-17T11:58:22.712 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:22 smithi165 ceph-mon[25702]: pgmap v11: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 7.2 MiB/s wr, 693 op/s 2024-09-17T11:58:22.712 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:22 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T11:58:22.923 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:22 smithi045 ceph-mon[23737]: pgmap v11: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 7.2 MiB/s wr, 693 op/s 2024-09-17T11:58:22.924 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:22 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T11:58:23.690 INFO:teuthology.orchestra.run.smithi045.stdout: 5 62053 11.31 MB/sec execute 24 sec latency 32.272 ms 2024-09-17T11:58:23.935 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:23 smithi045 ceph-mon[23737]: pgmap v12: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 7.3 MiB/s wr, 692 op/s 2024-09-17T11:58:24.219 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:23 smithi165 ceph-mon[25702]: pgmap v12: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 7.3 MiB/s wr, 692 op/s 2024-09-17T11:58:24.690 INFO:teuthology.orchestra.run.smithi045.stdout: 5 62577 11.21 MB/sec execute 25 sec latency 60.664 ms 2024-09-17T11:58:25.464 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.464 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.464 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:25.465 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:25.465 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:25.465 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:25.465 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:25.465 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:25.465 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:25.465 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:25.465 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: Adjusting osd_memory_target on smithi165 to 2704M 2024-09-17T11:58:25.465 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.465 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:58:25.465 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:58:25.466 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-17T11:58:25.466 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: Updating smithi165:/etc/ceph/ceph.conf 2024-09-17T11:58:25.466 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: Updating smithi165:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T11:58:25.466 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: Updating smithi045:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T11:58:25.466 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: Updating smithi165:/etc/ceph/ceph.client.admin.keyring 2024-09-17T11:58:25.466 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2024-09-17T11:58:25.466 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.466 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.466 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.466 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.466 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.466 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.466 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.466 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.466 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.467 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:25 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:25.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:25.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:25.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:25.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:25.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:25.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:25.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T11:58:25.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: Adjusting osd_memory_target on smithi165 to 2704M 2024-09-17T11:58:25.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:58:25.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:58:25.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-17T11:58:25.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: Updating smithi165:/etc/ceph/ceph.conf 2024-09-17T11:58:25.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: Updating smithi165:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T11:58:25.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: Updating smithi045:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T11:58:25.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: Updating smithi165:/etc/ceph/ceph.client.admin.keyring 2024-09-17T11:58:25.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2024-09-17T11:58:25.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:25 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:25.690 INFO:teuthology.orchestra.run.smithi045.stdout: 5 62911 11.21 MB/sec execute 26 sec latency 55.129 ms 2024-09-17T11:58:26.481 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:26 smithi045 ceph-mon[23737]: Updating smithi165:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T11:58:26.481 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:26 smithi045 ceph-mon[23737]: Updating smithi045:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T11:58:26.481 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:26 smithi045 ceph-mon[23737]: Reconfiguring alertmanager.smithi045 (dependencies changed)... 2024-09-17T11:58:26.481 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:26 smithi045 ceph-mon[23737]: Deploying daemon alertmanager.smithi045 on smithi045 2024-09-17T11:58:26.481 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:26 smithi045 ceph-mon[23737]: pgmap v13: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 502 op/s 2024-09-17T11:58:26.690 INFO:teuthology.orchestra.run.smithi045.stdout: 5 63292 11.35 MB/sec execute 27 sec latency 125.366 ms 2024-09-17T11:58:26.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:26 smithi165 ceph-mon[25702]: Updating smithi165:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T11:58:26.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:26 smithi165 ceph-mon[25702]: Updating smithi045:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T11:58:26.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:26 smithi165 ceph-mon[25702]: Reconfiguring alertmanager.smithi045 (dependencies changed)... 2024-09-17T11:58:26.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:26 smithi165 ceph-mon[25702]: Deploying daemon alertmanager.smithi045 on smithi045 2024-09-17T11:58:26.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:26 smithi165 ceph-mon[25702]: pgmap v13: 97 pgs: 97 active+clean; 334 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 502 op/s 2024-09-17T11:58:27.690 INFO:teuthology.orchestra.run.smithi045.stdout: 5 63732 11.58 MB/sec execute 28 sec latency 85.346 ms 2024-09-17T11:58:28.690 INFO:teuthology.orchestra.run.smithi045.stdout: 5 64122 11.64 MB/sec execute 29 sec latency 41.284 ms 2024-09-17T11:58:29.053 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:28 smithi045 ceph-mon[23737]: pgmap v14: 97 pgs: 97 active+clean; 335 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 833 op/s 2024-09-17T11:58:29.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:28 smithi165 ceph-mon[25702]: pgmap v14: 97 pgs: 97 active+clean; 335 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 833 op/s 2024-09-17T11:58:29.690 INFO:teuthology.orchestra.run.smithi045.stdout: 5 64417 11.45 MB/sec execute 30 sec latency 64.196 ms 2024-09-17T11:58:30.076 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:29 smithi045 ceph-mon[23737]: pgmap v15: 97 pgs: 97 active+clean; 335 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.6 MiB/s wr, 590 op/s 2024-09-17T11:58:30.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:29 smithi165 ceph-mon[25702]: pgmap v15: 97 pgs: 97 active+clean; 335 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.6 MiB/s wr, 590 op/s 2024-09-17T11:58:30.690 INFO:teuthology.orchestra.run.smithi045.stdout: 5 64659 11.17 MB/sec execute 31 sec latency 69.789 ms 2024-09-17T11:58:31.690 INFO:teuthology.orchestra.run.smithi045.stdout: 5 64907 10.99 MB/sec execute 32 sec latency 95.845 ms 2024-09-17T11:58:31.883 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:31 smithi045 ceph-mon[23737]: pgmap v16: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 7.3 MiB/s wr, 756 op/s 2024-09-17T11:58:32.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:31 smithi165 ceph-mon[25702]: pgmap v16: 97 pgs: 97 active+clean; 332 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 7.3 MiB/s wr, 756 op/s 2024-09-17T11:58:32.691 INFO:teuthology.orchestra.run.smithi045.stdout: 5 65194 10.85 MB/sec execute 33 sec latency 68.931 ms 2024-09-17T11:58:33.691 INFO:teuthology.orchestra.run.smithi045.stdout: 5 65592 10.81 MB/sec execute 34 sec latency 51.190 ms 2024-09-17T11:58:33.921 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:33 smithi045 ceph-mon[23737]: pgmap v17: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.4 MiB/s wr, 570 op/s 2024-09-17T11:58:34.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:33 smithi165 ceph-mon[25702]: pgmap v17: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.4 MiB/s wr, 570 op/s 2024-09-17T11:58:34.691 INFO:teuthology.orchestra.run.smithi045.stdout: 5 66154 10.83 MB/sec execute 35 sec latency 38.271 ms 2024-09-17T11:58:35.693 INFO:teuthology.orchestra.run.smithi045.stdout: 5 66557 10.84 MB/sec execute 36 sec latency 120.007 ms 2024-09-17T11:58:36.085 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:35 smithi045 ceph-mon[23737]: pgmap v18: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.6 MiB/s wr, 530 op/s 2024-09-17T11:58:36.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:35 smithi165 ceph-mon[25702]: pgmap v18: 97 pgs: 97 active+clean; 332 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.6 MiB/s wr, 530 op/s 2024-09-17T11:58:36.503 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T11:58:36.691 INFO:teuthology.orchestra.run.smithi045.stdout: 5 66960 11.00 MB/sec execute 37 sec latency 66.000 ms 2024-09-17T11:58:36.847 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:36 smithi045 ceph-mon[23737]: from='client.14722 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:58:37.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:36 smithi165 ceph-mon[25702]: from='client.14722 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:58:37.529 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T11:58:37.529 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (7m) 20s ago 9m 25.7M - 0.23.0 ba2b418f427c c97407c70fb6 2024-09-17T11:58:37.529 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (9m) 20s ago 9m 6413k - 17.2.0 e1d6a67b021e 8c510943b55e 2024-09-17T11:58:37.529 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (8m) 17s ago 8m 6409k - 17.2.0 e1d6a67b021e 687c24a87b70 2024-09-17T11:58:37.530 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (7m) 20s ago 8m 54.4M - dad864ee21e9 7d87d4101e56 2024-09-17T11:58:37.530 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (3m) 20s ago 3m 35.1M - 17.2.0 e1d6a67b021e 6a2a4a998dd1 2024-09-17T11:58:37.530 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (3m) 17s ago 3m 14.2M - 17.2.0 e1d6a67b021e d4c2987b9bac 2024-09-17T11:58:37.530 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:9283 running (10m) 20s ago 10m 220M - 17.2.0 e1d6a67b021e 5f496de39b08 2024-09-17T11:58:37.530 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283 running (42s) 17s ago 8m 495M - 19.3.0-4937-g74984932 6c1fcab85602 96e7a3af0643 2024-09-17T11:58:37.530 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (10m) 20s ago 10m 52.3M 2048M 17.2.0 e1d6a67b021e c4256bdf51b7 2024-09-17T11:58:37.530 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (8m) 17s ago 8m 38.7M 2048M 17.2.0 e1d6a67b021e 9219263d3aa8 2024-09-17T11:58:37.530 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (2m) 20s ago 2m 250M - 4.0 e1d6a67b021e b8fc749ebb71 2024-09-17T11:58:37.530 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (2m) 17s ago 2m 75.4M - 4.0 e1d6a67b021e 771b27764432 2024-09-17T11:58:37.530 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (8m) 20s ago 8m 20.7M - 1.3.1 1dbe0e931976 09dea232cb6b 2024-09-17T11:58:37.530 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (8m) 17s ago 8m 22.3M - 1.3.1 1dbe0e931976 fdb9449adb13 2024-09-17T11:58:37.530 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (7m) 20s ago 7m 275M 1937M 17.2.0 e1d6a67b021e e0ce4d29dc64 2024-09-17T11:58:37.530 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (7m) 20s ago 7m 198M 1937M 17.2.0 e1d6a67b021e da8bda7b47b3 2024-09-17T11:58:37.531 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (6m) 20s ago 6m 163M 1937M 17.2.0 e1d6a67b021e 21fe0d9e5ced 2024-09-17T11:58:37.531 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (6m) 20s ago 6m 215M 1937M 17.2.0 e1d6a67b021e 92a2389488e4 2024-09-17T11:58:37.531 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (5m) 17s ago 5m 236M 2704M 17.2.0 e1d6a67b021e e29eb74df25a 2024-09-17T11:58:37.531 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (5m) 17s ago 5m 235M 2704M 17.2.0 e1d6a67b021e 5d900eb9165b 2024-09-17T11:58:37.531 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (5m) 17s ago 5m 225M 2704M 17.2.0 e1d6a67b021e fae3e1bae466 2024-09-17T11:58:37.531 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (4m) 17s ago 4m 325M 2704M 17.2.0 e1d6a67b021e 71b9f2cdb810 2024-09-17T11:58:37.531 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (8m) 20s ago 8m 64.9M - 2.33.4 514e6a882f6e 528c529e18d6 2024-09-17T11:58:37.691 INFO:teuthology.orchestra.run.smithi045.stdout: 5 67357 11.16 MB/sec execute 38 sec latency 43.074 ms 2024-09-17T11:58:37.855 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:37 smithi045 ceph-mon[23737]: from='client.14726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:58:37.855 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:37 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T11:58:37.855 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:37 smithi045 ceph-mon[23737]: from='client.14730 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:58:37.855 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:37 smithi045 ceph-mon[23737]: pgmap v19: 97 pgs: 97 active+clean; 329 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 771 op/s 2024-09-17T11:58:38.187 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T11:58:38.187 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T11:58:38.187 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T11:58:38.187 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:58:38.187 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T11:58:38.187 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T11:58:38.187 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 1 2024-09-17T11:58:38.187 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:58:38.187 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T11:58:38.187 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T11:58:38.188 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:58:38.188 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T11:58:38.188 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T11:58:38.188 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:58:38.188 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T11:58:38.188 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-17T11:58:38.188 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 1 2024-09-17T11:58:38.188 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T11:58:38.188 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T11:58:38.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:37 smithi165 ceph-mon[25702]: from='client.14726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:58:38.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:37 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T11:58:38.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:37 smithi165 ceph-mon[25702]: from='client.14730 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:58:38.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:37 smithi165 ceph-mon[25702]: pgmap v19: 97 pgs: 97 active+clean; 329 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 771 op/s 2024-09-17T11:58:38.691 INFO:teuthology.orchestra.run.smithi045.stdout: 5 67686 11.11 MB/sec execute 39 sec latency 39.431 ms 2024-09-17T11:58:38.698 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T11:58:38.699 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T11:58:38.699 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T11:58:38.699 INFO:teuthology.orchestra.run.smithi045.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T11:58:38.699 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [], 2024-09-17T11:58:38.699 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "1/23 daemons upgraded", 2024-09-17T11:58:38.699 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "", 2024-09-17T11:58:38.699 INFO:teuthology.orchestra.run.smithi045.stdout: "is_paused": false 2024-09-17T11:58:38.699 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T11:58:38.867 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:38 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/2505409565' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T11:58:38.867 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:38 smithi045 ceph-mon[23737]: from='client.14738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:58:39.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:38 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/2505409565' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T11:58:39.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:38 smithi165 ceph-mon[25702]: from='client.14738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:58:39.286 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_OK 2024-09-17T11:58:39.691 INFO:teuthology.orchestra.run.smithi045.stdout: 5 67988 10.97 MB/sec execute 40 sec latency 56.237 ms 2024-09-17T11:58:39.890 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:39 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/851190770' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T11:58:39.890 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:39 smithi045 ceph-mon[23737]: pgmap v20: 97 pgs: 97 active+clean; 329 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 440 op/s 2024-09-17T11:58:40.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:39 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/851190770' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T11:58:40.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:39 smithi165 ceph-mon[25702]: pgmap v20: 97 pgs: 97 active+clean; 329 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 440 op/s 2024-09-17T11:58:40.691 INFO:teuthology.orchestra.run.smithi045.stdout: 5 68400 10.90 MB/sec execute 41 sec latency 57.511 ms 2024-09-17T11:58:41.691 INFO:teuthology.orchestra.run.smithi045.stdout: 5 68862 10.88 MB/sec execute 42 sec latency 34.904 ms 2024-09-17T11:58:42.421 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:42 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:42.421 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:42 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:42.421 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:42 smithi045 ceph-mon[23737]: Reconfiguring grafana.smithi045 (dependencies changed)... 2024-09-17T11:58:42.421 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:42 smithi045 ceph-mon[23737]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-17T11:58:42.421 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:42 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:42.421 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:42 smithi045 ceph-mon[23737]: pgmap v21: 97 pgs: 97 active+clean; 324 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 6.1 MiB/s wr, 586 op/s 2024-09-17T11:58:42.421 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:42 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:42.422 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:42 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T11:58:42.422 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:42 smithi045 ceph-mon[23737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T11:58:42.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:42 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:42.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:42 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:42.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:42 smithi165 ceph-mon[25702]: Reconfiguring grafana.smithi045 (dependencies changed)... 2024-09-17T11:58:42.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:42 smithi165 ceph-mon[25702]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-17T11:58:42.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:42 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:42.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:42 smithi165 ceph-mon[25702]: pgmap v21: 97 pgs: 97 active+clean; 324 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 6.1 MiB/s wr, 586 op/s 2024-09-17T11:58:42.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:42 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:42.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:42 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T11:58:42.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:42 smithi165 ceph-mon[25702]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T11:58:42.691 INFO:teuthology.orchestra.run.smithi045.stdout: 5 69356 10.82 MB/sec execute 43 sec latency 53.614 ms 2024-09-17T11:58:43.691 INFO:teuthology.orchestra.run.smithi045.stdout: 5 69773 10.87 MB/sec execute 44 sec latency 43.273 ms 2024-09-17T11:58:43.830 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:43 smithi165 ceph-mon[25702]: Reconfiguring daemon grafana.smithi045 on smithi045 2024-09-17T11:58:43.953 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:43 smithi045 ceph-mon[23737]: Reconfiguring daemon grafana.smithi045 on smithi045 2024-09-17T11:58:44.691 INFO:teuthology.orchestra.run.smithi045.stdout: 5 70350 11.10 MB/sec execute 45 sec latency 40.884 ms 2024-09-17T11:58:44.979 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:44 smithi045 ceph-mon[23737]: pgmap v22: 97 pgs: 97 active+clean; 326 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.2 MiB/s wr, 507 op/s 2024-09-17T11:58:45.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:44 smithi165 ceph-mon[25702]: pgmap v22: 97 pgs: 97 active+clean; 326 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.2 MiB/s wr, 507 op/s 2024-09-17T11:58:45.691 INFO:teuthology.orchestra.run.smithi045.stdout: 5 70985 11.31 MB/sec execute 46 sec latency 44.074 ms 2024-09-17T11:58:46.691 INFO:teuthology.orchestra.run.smithi045.stdout: 5 71379 11.26 MB/sec execute 47 sec latency 55.948 ms 2024-09-17T11:58:46.892 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:46 smithi045 ceph-mon[23737]: pgmap v23: 97 pgs: 97 active+clean; 326 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.7 MiB/s wr, 473 op/s 2024-09-17T11:58:47.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:46 smithi165 ceph-mon[25702]: pgmap v23: 97 pgs: 97 active+clean; 326 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.7 MiB/s wr, 473 op/s 2024-09-17T11:58:47.691 INFO:teuthology.orchestra.run.smithi045.stdout: 5 71708 11.18 MB/sec execute 48 sec latency 43.574 ms 2024-09-17T11:58:48.691 INFO:teuthology.orchestra.run.smithi045.stdout: 5 72018 11.15 MB/sec execute 49 sec latency 38.393 ms 2024-09-17T11:58:48.786 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:48 smithi045 ceph-mon[23737]: pgmap v24: 97 pgs: 97 active+clean; 333 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 8.0 MiB/s wr, 766 op/s 2024-09-17T11:58:49.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:48 smithi165 ceph-mon[25702]: pgmap v24: 97 pgs: 97 active+clean; 333 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 8.0 MiB/s wr, 766 op/s 2024-09-17T11:58:49.691 INFO:teuthology.orchestra.run.smithi045.stdout: 5 72374 11.05 MB/sec execute 50 sec latency 54.226 ms 2024-09-17T11:58:50.301 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:50 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:50.301 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:50 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:50.301 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:50 smithi045 ceph-mon[23737]: Reconfiguring node-exporter.smithi045 (dependencies changed)... 2024-09-17T11:58:50.301 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:50 smithi045 ceph-mon[23737]: Deploying daemon node-exporter.smithi045 on smithi045 2024-09-17T11:58:50.301 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:50 smithi045 ceph-mon[23737]: pgmap v25: 97 pgs: 97 active+clean; 333 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.6 MiB/s wr, 525 op/s 2024-09-17T11:58:50.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:50 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:50.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:50 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:50.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:50 smithi165 ceph-mon[25702]: Reconfiguring node-exporter.smithi045 (dependencies changed)... 2024-09-17T11:58:50.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:50 smithi165 ceph-mon[25702]: Deploying daemon node-exporter.smithi045 on smithi045 2024-09-17T11:58:50.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:50 smithi165 ceph-mon[25702]: pgmap v25: 97 pgs: 97 active+clean; 333 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.6 MiB/s wr, 525 op/s 2024-09-17T11:58:50.691 INFO:teuthology.orchestra.run.smithi045.stdout: 5 72784 10.96 MB/sec execute 51 sec latency 57.735 ms 2024-09-17T11:58:51.692 INFO:teuthology.orchestra.run.smithi045.stdout: 5 73122 10.84 MB/sec execute 52 sec latency 58.614 ms 2024-09-17T11:58:51.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:51 smithi045 ceph-mon[23737]: pgmap v26: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.6 MiB/s wr, 647 op/s 2024-09-17T11:58:52.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:51 smithi165 ceph-mon[25702]: pgmap v26: 97 pgs: 97 active+clean; 326 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.6 MiB/s wr, 647 op/s 2024-09-17T11:58:52.692 INFO:teuthology.orchestra.run.smithi045.stdout: 5 73523 10.95 MB/sec execute 53 sec latency 39.285 ms 2024-09-17T11:58:52.868 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:52 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T11:58:53.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:52 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T11:58:53.692 INFO:teuthology.orchestra.run.smithi045.stdout: 5 74126 11.15 MB/sec execute 54 sec latency 36.290 ms 2024-09-17T11:58:53.888 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:53 smithi045 ceph-mon[23737]: pgmap v27: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.8 MiB/s wr, 594 op/s 2024-09-17T11:58:54.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:53 smithi165 ceph-mon[25702]: pgmap v27: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.8 MiB/s wr, 594 op/s 2024-09-17T11:58:54.692 INFO:teuthology.orchestra.run.smithi045.stdout: 5 74434 11.11 MB/sec execute 55 sec latency 55.453 ms 2024-09-17T11:58:55.692 INFO:teuthology.orchestra.run.smithi045.stdout: 5 74768 11.10 MB/sec execute 56 sec latency 51.154 ms 2024-09-17T11:58:55.915 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:55 smithi045 ceph-mon[23737]: pgmap v28: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.0 MiB/s wr, 508 op/s 2024-09-17T11:58:56.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:55 smithi165 ceph-mon[25702]: pgmap v28: 97 pgs: 97 active+clean; 323 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.0 MiB/s wr, 508 op/s 2024-09-17T11:58:56.692 INFO:teuthology.orchestra.run.smithi045.stdout: 5 75090 11.00 MB/sec execute 57 sec latency 94.087 ms 2024-09-17T11:58:57.692 INFO:teuthology.orchestra.run.smithi045.stdout: 5 75399 10.93 MB/sec execute 58 sec latency 58.437 ms 2024-09-17T11:58:58.692 INFO:teuthology.orchestra.run.smithi045.stdout: 5 75727 10.91 MB/sec execute 59 sec latency 49.280 ms 2024-09-17T11:58:58.734 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:58 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:58.734 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:58 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:58.734 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:58 smithi045 ceph-mon[23737]: Reconfiguring prometheus.smithi045 (dependencies changed)... 2024-09-17T11:58:58.734 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:58 smithi045 ceph-mon[23737]: pgmap v29: 97 pgs: 97 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 736 op/s 2024-09-17T11:58:59.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:58 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:59.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:58 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:58:59.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:58 smithi165 ceph-mon[25702]: Reconfiguring prometheus.smithi045 (dependencies changed)... 2024-09-17T11:58:59.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:58 smithi165 ceph-mon[25702]: pgmap v29: 97 pgs: 97 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 736 op/s 2024-09-17T11:58:59.692 INFO:teuthology.orchestra.run.smithi045.stdout: 5 76251 10.90 MB/sec execute 60 sec latency 43.321 ms 2024-09-17T11:58:59.754 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:58:59 smithi045 ceph-mon[23737]: Deploying daemon prometheus.smithi045 on smithi045 2024-09-17T11:59:00.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:58:59 smithi165 ceph-mon[25702]: Deploying daemon prometheus.smithi045 on smithi045 2024-09-17T11:59:00.692 INFO:teuthology.orchestra.run.smithi045.stdout: 5 76619 10.80 MB/sec execute 61 sec latency 44.239 ms 2024-09-17T11:59:00.778 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:00 smithi045 ceph-mon[23737]: pgmap v30: 97 pgs: 97 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 442 op/s 2024-09-17T11:59:01.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:00 smithi165 ceph-mon[25702]: pgmap v30: 97 pgs: 97 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.5 MiB/s wr, 442 op/s 2024-09-17T11:59:01.692 INFO:teuthology.orchestra.run.smithi045.stdout: 5 76971 10.84 MB/sec execute 62 sec latency 42.295 ms 2024-09-17T11:59:02.066 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:01 smithi045 ceph-mon[23737]: pgmap v31: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 6.0 MiB/s wr, 615 op/s 2024-09-17T11:59:02.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:01 smithi165 ceph-mon[25702]: pgmap v31: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 6.0 MiB/s wr, 615 op/s 2024-09-17T11:59:02.692 INFO:teuthology.orchestra.run.smithi045.stdout: 5 77370 10.92 MB/sec execute 63 sec latency 41.752 ms 2024-09-17T11:59:03.692 INFO:teuthology.orchestra.run.smithi045.stdout: 5 77830 11.00 MB/sec execute 64 sec latency 37.294 ms 2024-09-17T11:59:03.962 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:03 smithi045 ceph-mon[23737]: pgmap v32: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.6 MiB/s wr, 557 op/s 2024-09-17T11:59:04.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:03 smithi165 ceph-mon[25702]: pgmap v32: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.6 MiB/s wr, 557 op/s 2024-09-17T11:59:04.692 INFO:teuthology.orchestra.run.smithi045.stdout: 5 78078 10.95 MB/sec execute 65 sec latency 58.484 ms 2024-09-17T11:59:05.692 INFO:teuthology.orchestra.run.smithi045.stdout: 5 78368 10.88 MB/sec execute 66 sec latency 52.080 ms 2024-09-17T11:59:05.987 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:05 smithi045 ceph-mon[23737]: pgmap v33: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.9 MiB/s wr, 464 op/s 2024-09-17T11:59:06.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:05 smithi165 ceph-mon[25702]: pgmap v33: 97 pgs: 97 active+clean; 325 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.9 MiB/s wr, 464 op/s 2024-09-17T11:59:06.692 INFO:teuthology.orchestra.run.smithi045.stdout: 5 78639 10.78 MB/sec execute 67 sec latency 62.698 ms 2024-09-17T11:59:07.498 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:07 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T11:59:07.692 INFO:teuthology.orchestra.run.smithi045.stdout: 5 78965 10.73 MB/sec execute 68 sec latency 42.968 ms 2024-09-17T11:59:07.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:07 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T11:59:08.502 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:08 smithi045 ceph-mon[23737]: pgmap v34: 97 pgs: 97 active+clean; 332 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.1 MiB/s wr, 656 op/s 2024-09-17T11:59:08.693 INFO:teuthology.orchestra.run.smithi045.stdout: 5 79357 10.73 MB/sec execute 69 sec latency 40.201 ms 2024-09-17T11:59:08.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:08 smithi165 ceph-mon[25702]: pgmap v34: 97 pgs: 97 active+clean; 332 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 7.1 MiB/s wr, 656 op/s 2024-09-17T11:59:09.692 INFO:teuthology.orchestra.run.smithi045.stdout: 5 79885 10.72 MB/sec execute 70 sec latency 33.325 ms 2024-09-17T11:59:09.904 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T11:59:10.011 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:09 smithi045 ceph-mon[23737]: pgmap v35: 97 pgs: 97 active+clean; 332 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 428 op/s 2024-09-17T11:59:10.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:09 smithi165 ceph-mon[25702]: pgmap v35: 97 pgs: 97 active+clean; 332 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 428 op/s 2024-09-17T11:59:10.693 INFO:teuthology.orchestra.run.smithi045.stdout: 5 80385 10.77 MB/sec execute 71 sec latency 32.124 ms 2024-09-17T11:59:11.027 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:10 smithi045 ceph-mon[23737]: from='client.24523 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:59:11.027 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:10 smithi045 ceph-mon[23737]: from='client.24525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:59:11.069 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T11:59:11.069 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 starting - - - - 2024-09-17T11:59:11.069 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (9m) 53s ago 9m 6413k - 17.2.0 e1d6a67b021e 8c510943b55e 2024-09-17T11:59:11.069 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (9m) 51s ago 9m 6409k - 17.2.0 e1d6a67b021e 687c24a87b70 2024-09-17T11:59:11.069 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 starting - - - - 2024-09-17T11:59:11.069 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (3m) 53s ago 3m 35.1M - 17.2.0 e1d6a67b021e 6a2a4a998dd1 2024-09-17T11:59:11.069 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (3m) 51s ago 3m 14.2M - 17.2.0 e1d6a67b021e d4c2987b9bac 2024-09-17T11:59:11.069 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:9283 running (11m) 53s ago 11m 220M - 17.2.0 e1d6a67b021e 5f496de39b08 2024-09-17T11:59:11.069 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283 running (75s) 51s ago 9m 495M - 19.3.0-4937-g74984932 6c1fcab85602 96e7a3af0643 2024-09-17T11:59:11.069 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (11m) 53s ago 11m 52.3M 2048M 17.2.0 e1d6a67b021e c4256bdf51b7 2024-09-17T11:59:11.069 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (9m) 51s ago 9m 38.7M 2048M 17.2.0 e1d6a67b021e 9219263d3aa8 2024-09-17T11:59:11.070 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (3m) 53s ago 3m 250M - 4.0 e1d6a67b021e b8fc749ebb71 2024-09-17T11:59:11.070 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (3m) 51s ago 3m 75.4M - 4.0 e1d6a67b021e 771b27764432 2024-09-17T11:59:11.070 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 starting - - - - 2024-09-17T11:59:11.070 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (8m) 51s ago 8m 22.3M - 1.3.1 1dbe0e931976 fdb9449adb13 2024-09-17T11:59:11.070 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (8m) 53s ago 8m 275M 1937M 17.2.0 e1d6a67b021e e0ce4d29dc64 2024-09-17T11:59:11.070 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (7m) 53s ago 7m 198M 1937M 17.2.0 e1d6a67b021e da8bda7b47b3 2024-09-17T11:59:11.070 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (7m) 53s ago 7m 163M 1937M 17.2.0 e1d6a67b021e 21fe0d9e5ced 2024-09-17T11:59:11.070 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (6m) 53s ago 6m 215M 1937M 17.2.0 e1d6a67b021e 92a2389488e4 2024-09-17T11:59:11.070 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (6m) 51s ago 6m 236M 2704M 17.2.0 e1d6a67b021e e29eb74df25a 2024-09-17T11:59:11.070 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (6m) 51s ago 6m 235M 2704M 17.2.0 e1d6a67b021e 5d900eb9165b 2024-09-17T11:59:11.070 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (5m) 51s ago 5m 225M 2704M 17.2.0 e1d6a67b021e fae3e1bae466 2024-09-17T11:59:11.070 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (5m) 51s ago 5m 325M 2704M 17.2.0 e1d6a67b021e 71b9f2cdb810 2024-09-17T11:59:11.070 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (8m) 53s ago 8m 64.9M - 2.33.4 514e6a882f6e 528c529e18d6 2024-09-17T11:59:11.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:10 smithi165 ceph-mon[25702]: from='client.24523 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:59:11.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:10 smithi165 ceph-mon[25702]: from='client.24525 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:59:11.692 INFO:teuthology.orchestra.run.smithi045.stdout: 5 80960 10.93 MB/sec execute 72 sec latency 26.112 ms 2024-09-17T11:59:11.815 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T11:59:11.815 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T11:59:11.815 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T11:59:11.815 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:59:11.815 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T11:59:11.815 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T11:59:11.815 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 1 2024-09-17T11:59:11.815 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:59:11.816 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T11:59:11.816 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T11:59:11.816 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:59:11.816 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T11:59:11.816 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T11:59:11.816 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:59:11.816 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T11:59:11.816 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-17T11:59:11.816 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 1 2024-09-17T11:59:11.816 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T11:59:11.816 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T11:59:12.037 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:11 smithi045 ceph-mon[23737]: from='client.14754 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:59:12.037 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:11 smithi045 ceph-mon[23737]: pgmap v36: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.8 MiB/s wr, 526 op/s 2024-09-17T11:59:12.037 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:11 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/1533670018' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T11:59:12.166 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:11 smithi165 ceph-mon[25702]: from='client.14754 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:59:12.166 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:11 smithi165 ceph-mon[25702]: pgmap v36: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.8 MiB/s wr, 526 op/s 2024-09-17T11:59:12.167 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:11 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/1533670018' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T11:59:12.424 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T11:59:12.424 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T11:59:12.424 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T11:59:12.424 INFO:teuthology.orchestra.run.smithi045.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T11:59:12.424 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [], 2024-09-17T11:59:12.424 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "1/23 daemons upgraded", 2024-09-17T11:59:12.425 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "", 2024-09-17T11:59:12.425 INFO:teuthology.orchestra.run.smithi045.stdout: "is_paused": false 2024-09-17T11:59:12.425 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T11:59:12.692 INFO:teuthology.orchestra.run.smithi045.stdout: 5 81425 10.97 MB/sec execute 73 sec latency 65.751 ms 2024-09-17T11:59:13.061 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:12 smithi045 ceph-mon[23737]: from='client.14762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:59:13.140 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_OK 2024-09-17T11:59:13.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:12 smithi165 ceph-mon[25702]: from='client.14762 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:59:13.692 INFO:teuthology.orchestra.run.smithi045.stdout: 5 81674 10.91 MB/sec execute 74 sec latency 74.280 ms 2024-09-17T11:59:14.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:13 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/2415153112' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T11:59:14.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:13 smithi045 ceph-mon[23737]: pgmap v37: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.6 MiB/s wr, 532 op/s 2024-09-17T11:59:14.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:13 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/2415153112' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T11:59:14.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:13 smithi165 ceph-mon[25702]: pgmap v37: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.6 MiB/s wr, 532 op/s 2024-09-17T11:59:14.693 INFO:teuthology.orchestra.run.smithi045.stdout: 5 81889 10.81 MB/sec execute 75 sec latency 60.257 ms 2024-09-17T11:59:15.692 INFO:teuthology.orchestra.run.smithi045.stdout: 5 82133 10.74 MB/sec execute 76 sec latency 92.717 ms 2024-09-17T11:59:15.910 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:15 smithi045 ceph-mon[23737]: pgmap v38: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.9 MiB/s wr, 467 op/s 2024-09-17T11:59:16.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:15 smithi165 ceph-mon[25702]: pgmap v38: 97 pgs: 97 active+clean; 330 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.9 MiB/s wr, 467 op/s 2024-09-17T11:59:16.693 INFO:teuthology.orchestra.run.smithi045.stdout: 5 82462 10.71 MB/sec execute 77 sec latency 63.070 ms 2024-09-17T11:59:17.693 INFO:teuthology.orchestra.run.smithi045.stdout: 5 82801 10.70 MB/sec execute 78 sec latency 52.088 ms 2024-09-17T11:59:17.937 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:17 smithi045 ceph-mon[23737]: pgmap v39: 97 pgs: 97 active+clean; 328 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 656 op/s 2024-09-17T11:59:18.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:17 smithi165 ceph-mon[25702]: pgmap v39: 97 pgs: 97 active+clean; 328 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 656 op/s 2024-09-17T11:59:18.693 INFO:teuthology.orchestra.run.smithi045.stdout: 5 83212 10.68 MB/sec execute 79 sec latency 39.676 ms 2024-09-17T11:59:19.693 INFO:teuthology.orchestra.run.smithi045.stdout: 5 83598 10.60 MB/sec execute 80 sec latency 55.786 ms 2024-09-17T11:59:19.970 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:19 smithi045 ceph-mon[23737]: pgmap v40: 97 pgs: 97 active+clean; 328 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.7 MiB/s wr, 464 op/s 2024-09-17T11:59:20.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:19 smithi165 ceph-mon[25702]: pgmap v40: 97 pgs: 97 active+clean; 328 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.7 MiB/s wr, 464 op/s 2024-09-17T11:59:20.693 INFO:teuthology.orchestra.run.smithi045.stdout: 5 84078 10.69 MB/sec execute 81 sec latency 76.815 ms 2024-09-17T11:59:21.693 INFO:teuthology.orchestra.run.smithi045.stdout: 5 84652 10.82 MB/sec execute 82 sec latency 54.161 ms 2024-09-17T11:59:21.931 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:21 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:21.931 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:21 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:21.932 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:21 smithi165 ceph-mon[25702]: Reconfiguring node-exporter.smithi165 (dependencies changed)... 2024-09-17T11:59:21.932 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:21 smithi165 ceph-mon[25702]: Deploying daemon node-exporter.smithi165 on smithi165 2024-09-17T11:59:21.932 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:21 smithi165 ceph-mon[25702]: pgmap v41: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.9 MiB/s wr, 591 op/s 2024-09-17T11:59:21.994 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:21 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:21.994 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:21 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:21.994 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:21 smithi045 ceph-mon[23737]: Reconfiguring node-exporter.smithi165 (dependencies changed)... 2024-09-17T11:59:21.994 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:21 smithi045 ceph-mon[23737]: Deploying daemon node-exporter.smithi165 on smithi165 2024-09-17T11:59:21.994 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:21 smithi045 ceph-mon[23737]: pgmap v41: 97 pgs: 97 active+clean; 326 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.9 MiB/s wr, 591 op/s 2024-09-17T11:59:22.693 INFO:teuthology.orchestra.run.smithi045.stdout: 5 85048 10.82 MB/sec execute 83 sec latency 43.759 ms 2024-09-17T11:59:23.016 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:22 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T11:59:23.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:22 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T11:59:23.693 INFO:teuthology.orchestra.run.smithi045.stdout: 5 85359 10.77 MB/sec execute 84 sec latency 48.603 ms 2024-09-17T11:59:24.020 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:23 smithi045 ceph-mon[23737]: pgmap v42: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.8 MiB/s wr, 616 op/s 2024-09-17T11:59:24.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:23 smithi165 ceph-mon[25702]: pgmap v42: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.8 MiB/s wr, 616 op/s 2024-09-17T11:59:24.693 INFO:teuthology.orchestra.run.smithi045.stdout: 5 85652 10.72 MB/sec execute 85 sec latency 52.841 ms 2024-09-17T11:59:25.693 INFO:teuthology.orchestra.run.smithi045.stdout: 5 85996 10.68 MB/sec execute 86 sec latency 32.388 ms 2024-09-17T11:59:26.054 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:25 smithi045 ceph-mon[23737]: pgmap v43: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 438 op/s 2024-09-17T11:59:26.096 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:25 smithi165 ceph-mon[25702]: pgmap v43: 97 pgs: 97 active+clean; 331 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 438 op/s 2024-09-17T11:59:26.693 INFO:teuthology.orchestra.run.smithi045.stdout: 5 86423 10.72 MB/sec execute 87 sec latency 35.660 ms 2024-09-17T11:59:27.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:27 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:27.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:27 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:27.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:27 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T11:59:27.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:27 smithi165 ceph-mon[25702]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T11:59:27.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:27 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi045.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T11:59:27.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:27 smithi165 ceph-mon[25702]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi045.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T11:59:27.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:27 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:27.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:27 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T11:59:27.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:27 smithi165 ceph-mon[25702]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T11:59:27.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:27 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi045.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T11:59:27.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:27 smithi165 ceph-mon[25702]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi045.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T11:59:27.268 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:27 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:27.268 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:27 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T11:59:27.268 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:27 smithi165 ceph-mon[25702]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T11:59:27.268 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:27 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi045.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T11:59:27.268 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:27 smithi165 ceph-mon[25702]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi045.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T11:59:27.268 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:27 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:27.268 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:27 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T11:59:27.268 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:27 smithi165 ceph-mon[25702]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T11:59:27.268 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:27 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:59:27.268 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:27 smithi165 ceph-mon[25702]: Upgrade: Need to upgrade myself (mgr.smithi165.ykdszc) 2024-09-17T11:59:27.268 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:27 smithi165 ceph-mon[25702]: Upgrade: Need to upgrade myself (mgr.smithi165.ykdszc) 2024-09-17T11:59:27.268 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:27 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T11:59:27.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:27 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:27.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:27 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:27.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:27 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T11:59:27.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:27 smithi045 ceph-mon[23737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T11:59:27.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:27 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi045.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T11:59:27.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:27 smithi045 ceph-mon[23737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi045.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T11:59:27.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:27 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:27.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:27 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T11:59:27.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:27 smithi045 ceph-mon[23737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T11:59:27.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:27 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi045.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T11:59:27.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:27 smithi045 ceph-mon[23737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi045.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T11:59:27.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:27 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:27.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:27 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T11:59:27.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:27 smithi045 ceph-mon[23737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T11:59:27.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:27 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi045.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T11:59:27.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:27 smithi045 ceph-mon[23737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi045.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T11:59:27.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:27 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:27.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:27 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T11:59:27.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:27 smithi045 ceph-mon[23737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T11:59:27.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:27 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:59:27.342 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:27 smithi045 ceph-mon[23737]: Upgrade: Need to upgrade myself (mgr.smithi165.ykdszc) 2024-09-17T11:59:27.342 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:27 smithi045 ceph-mon[23737]: Upgrade: Need to upgrade myself (mgr.smithi165.ykdszc) 2024-09-17T11:59:27.342 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:27 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T11:59:27.693 INFO:teuthology.orchestra.run.smithi045.stdout: 5 86846 10.66 MB/sec execute 88 sec latency 37.303 ms 2024-09-17T11:59:28.093 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:28 smithi045 ceph-mon[23737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T11:59:28.093 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:28 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T11:59:28.093 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:28 smithi045 ceph-mon[23737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T11:59:28.093 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:28 smithi045 ceph-mon[23737]: pgmap v44: 97 pgs: 97 active+clean; 328 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 664 op/s 2024-09-17T11:59:28.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:28 smithi165 ceph-mon[25702]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T11:59:28.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:28 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T11:59:28.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:28 smithi165 ceph-mon[25702]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T11:59:28.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:28 smithi165 ceph-mon[25702]: pgmap v44: 97 pgs: 97 active+clean; 328 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 664 op/s 2024-09-17T11:59:28.693 INFO:teuthology.orchestra.run.smithi045.stdout: 5 87210 10.62 MB/sec execute 89 sec latency 44.644 ms 2024-09-17T11:59:29.693 INFO:teuthology.orchestra.run.smithi045.stdout: 5 87714 10.70 MB/sec execute 90 sec latency 33.137 ms 2024-09-17T11:59:30.371 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:30 smithi045 ceph-mon[23737]: Upgrade: Updating mgr.smithi045.jtwyrp 2024-09-17T11:59:30.371 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:30 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:30.371 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:30 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi045.jtwyrp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T11:59:30.371 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:30 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi045.jtwyrp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T11:59:30.371 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:30 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T11:59:30.371 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:30 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:59:30.372 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:30 smithi045 ceph-mon[23737]: Deploying daemon mgr.smithi045.jtwyrp on smithi045 2024-09-17T11:59:30.372 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:30 smithi045 ceph-mon[23737]: pgmap v45: 97 pgs: 97 active+clean; 328 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 475 op/s 2024-09-17T11:59:30.693 INFO:teuthology.orchestra.run.smithi045.stdout: 5 88179 10.81 MB/sec execute 91 sec latency 46.135 ms 2024-09-17T11:59:30.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:30 smithi165 ceph-mon[25702]: Upgrade: Updating mgr.smithi045.jtwyrp 2024-09-17T11:59:30.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:30 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:30.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:30 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi045.jtwyrp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T11:59:30.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:30 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi045.jtwyrp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T11:59:30.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:30 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T11:59:30.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:30 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:59:30.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:30 smithi165 ceph-mon[25702]: Deploying daemon mgr.smithi045.jtwyrp on smithi045 2024-09-17T11:59:30.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:30 smithi165 ceph-mon[25702]: pgmap v45: 97 pgs: 97 active+clean; 328 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 475 op/s 2024-09-17T11:59:31.693 INFO:teuthology.orchestra.run.smithi045.stdout: 5 88456 10.77 MB/sec execute 92 sec latency 45.049 ms 2024-09-17T11:59:31.936 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:31 smithi045 ceph-mon[23737]: pgmap v46: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.1 MiB/s wr, 638 op/s 2024-09-17T11:59:32.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:31 smithi165 ceph-mon[25702]: pgmap v46: 97 pgs: 97 active+clean; 329 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.1 MiB/s wr, 638 op/s 2024-09-17T11:59:32.693 INFO:teuthology.orchestra.run.smithi045.stdout: 5 88741 10.73 MB/sec execute 93 sec latency 42.579 ms 2024-09-17T11:59:33.693 INFO:teuthology.orchestra.run.smithi045.stdout: 5 88966 10.65 MB/sec execute 94 sec latency 52.969 ms 2024-09-17T11:59:34.012 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:33 smithi045 ceph-mon[23737]: pgmap v47: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.1 MiB/s wr, 578 op/s 2024-09-17T11:59:34.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:33 smithi165 ceph-mon[25702]: pgmap v47: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.1 MiB/s wr, 578 op/s 2024-09-17T11:59:34.694 INFO:teuthology.orchestra.run.smithi045.stdout: 5 89228 10.60 MB/sec execute 95 sec latency 68.573 ms 2024-09-17T11:59:35.694 INFO:teuthology.orchestra.run.smithi045.stdout: 5 89515 10.55 MB/sec execute 96 sec latency 67.770 ms 2024-09-17T11:59:36.063 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:35 smithi045 ceph-mon[23737]: pgmap v48: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 455 op/s 2024-09-17T11:59:36.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:35 smithi165 ceph-mon[25702]: pgmap v48: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 455 op/s 2024-09-17T11:59:36.694 INFO:teuthology.orchestra.run.smithi045.stdout: 5 89789 10.53 MB/sec execute 97 sec latency 68.726 ms 2024-09-17T11:59:37.072 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:36 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:37.072 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:36 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:37.072 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:36 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:59:37.073 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:36 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:37.073 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:36 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:37.238 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:36 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:37.238 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:36 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:37.238 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:36 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:59:37.238 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:36 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:37.238 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:36 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:37.694 INFO:teuthology.orchestra.run.smithi045.stdout: 5 90185 10.51 MB/sec execute 98 sec latency 42.520 ms 2024-09-17T11:59:38.105 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:37 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T11:59:38.105 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:37 smithi045 ceph-mon[23737]: pgmap v49: 97 pgs: 97 active+clean; 324 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.5 MiB/s wr, 634 op/s 2024-09-17T11:59:38.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:37 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T11:59:38.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:37 smithi165 ceph-mon[25702]: pgmap v49: 97 pgs: 97 active+clean; 324 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.5 MiB/s wr, 634 op/s 2024-09-17T11:59:38.704 INFO:teuthology.orchestra.run.smithi045.stdout: 5 90514 10.48 MB/sec execute 99 sec latency 55.267 ms 2024-09-17T11:59:39.704 INFO:teuthology.orchestra.run.smithi045.stdout: 5 90774 10.41 MB/sec execute 100 sec latency 52.154 ms 2024-09-17T11:59:40.174 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:39 smithi045 ceph-mon[23737]: mgrmap e28: smithi165.ykdszc(active, since 94s), standbys: smithi045.jtwyrp 2024-09-17T11:59:40.174 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:39 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:40.174 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:39 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:40.174 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:39 smithi045 ceph-mon[23737]: pgmap v50: 97 pgs: 97 active+clean; 324 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 408 op/s 2024-09-17T11:59:40.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:39 smithi165 ceph-mon[25702]: mgrmap e28: smithi165.ykdszc(active, since 94s), standbys: smithi045.jtwyrp 2024-09-17T11:59:40.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:39 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:40.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:39 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:40.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:39 smithi165 ceph-mon[25702]: pgmap v50: 97 pgs: 97 active+clean; 324 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 408 op/s 2024-09-17T11:59:40.704 INFO:teuthology.orchestra.run.smithi045.stdout: 5 91119 10.45 MB/sec execute 101 sec latency 49.260 ms 2024-09-17T11:59:41.704 INFO:teuthology.orchestra.run.smithi045.stdout: 5 91417 10.43 MB/sec execute 102 sec latency 72.486 ms 2024-09-17T11:59:41.970 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:41 smithi045 ceph-mon[23737]: pgmap v51: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.6 MiB/s wr, 564 op/s 2024-09-17T11:59:42.166 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:41 smithi165 ceph-mon[25702]: pgmap v51: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.6 MiB/s wr, 564 op/s 2024-09-17T11:59:42.704 INFO:teuthology.orchestra.run.smithi045.stdout: 5 91874 10.54 MB/sec execute 103 sec latency 48.016 ms 2024-09-17T11:59:43.669 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T11:59:43.704 INFO:teuthology.orchestra.run.smithi045.stdout: 5 92100 10.49 MB/sec execute 104 sec latency 101.590 ms 2024-09-17T11:59:44.013 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:43 smithi045 ceph-mon[23737]: Standby manager daemon smithi045.jtwyrp restarted 2024-09-17T11:59:44.013 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:43 smithi045 ceph-mon[23737]: Standby manager daemon smithi045.jtwyrp started 2024-09-17T11:59:44.013 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:43 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi045.jtwyrp/crt"}]: dispatch 2024-09-17T11:59:44.013 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:43 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T11:59:44.013 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:43 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi045.jtwyrp/key"}]: dispatch 2024-09-17T11:59:44.013 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:43 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T11:59:44.013 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:43 smithi045 ceph-mon[23737]: from='client.24535 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:59:44.013 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:43 smithi045 ceph-mon[23737]: pgmap v52: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.9 MiB/s wr, 437 op/s 2024-09-17T11:59:44.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:43 smithi165 ceph-mon[25702]: Standby manager daemon smithi045.jtwyrp restarted 2024-09-17T11:59:44.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:43 smithi165 ceph-mon[25702]: Standby manager daemon smithi045.jtwyrp started 2024-09-17T11:59:44.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:43 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi045.jtwyrp/crt"}]: dispatch 2024-09-17T11:59:44.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:43 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T11:59:44.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:43 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi045.jtwyrp/key"}]: dispatch 2024-09-17T11:59:44.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:43 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T11:59:44.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:43 smithi165 ceph-mon[25702]: from='client.24535 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:59:44.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:43 smithi165 ceph-mon[25702]: pgmap v52: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.9 MiB/s wr, 437 op/s 2024-09-17T11:59:44.552 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T11:59:44.552 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (64s) 0s ago 10m 21.1M - 0.27.0 11f11916f8cd 2a2a54402d2a 2024-09-17T11:59:44.553 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (10m) 0s ago 10m 6413k - 17.2.0 e1d6a67b021e 8c510943b55e 2024-09-17T11:59:44.553 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (9m) 4s ago 9m 6409k - 17.2.0 e1d6a67b021e 687c24a87b70 2024-09-17T11:59:44.553 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (56s) 0s ago 10m 50.7M - dad864ee21e9 371aaf55379f 2024-09-17T11:59:44.553 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (4m) 0s ago 4m 36.3M - 17.2.0 e1d6a67b021e 6a2a4a998dd1 2024-09-17T11:59:44.553 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (4m) 4s ago 4m 14.7M - 17.2.0 e1d6a67b021e d4c2987b9bac 2024-09-17T11:59:44.553 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:8443,9283,8765 running (9s) 0s ago 11m 89.7M - 19.3.0-4937-g74984932 6c1fcab85602 ae2f507bd67d 2024-09-17T11:59:44.553 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283 running (109s) 4s ago 9m 516M - 19.3.0-4937-g74984932 6c1fcab85602 96e7a3af0643 2024-09-17T11:59:44.553 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (11m) 0s ago 11m 55.0M 2048M 17.2.0 e1d6a67b021e c4256bdf51b7 2024-09-17T11:59:44.553 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (9m) 4s ago 9m 40.5M 2048M 17.2.0 e1d6a67b021e 9219263d3aa8 2024-09-17T11:59:44.553 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (3m) 0s ago 3m 258M - 4.0 e1d6a67b021e b8fc749ebb71 2024-09-17T11:59:44.553 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (3m) 4s ago 3m 81.5M - 4.0 e1d6a67b021e 771b27764432 2024-09-17T11:59:44.553 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (48s) 0s ago 10m 8316k - 1.7.0 72c9c2088986 989b5738cb69 2024-09-17T11:59:44.553 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (19s) 4s ago 9m 7323k - 1.7.0 72c9c2088986 e1e8ffab3f21 2024-09-17T11:59:44.554 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (8m) 0s ago 8m 379M 1937M 17.2.0 e1d6a67b021e e0ce4d29dc64 2024-09-17T11:59:44.554 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (8m) 0s ago 8m 270M 1937M 17.2.0 e1d6a67b021e da8bda7b47b3 2024-09-17T11:59:44.554 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (7m) 0s ago 7m 245M 1937M 17.2.0 e1d6a67b021e 21fe0d9e5ced 2024-09-17T11:59:44.554 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (7m) 0s ago 7m 339M 1937M 17.2.0 e1d6a67b021e 92a2389488e4 2024-09-17T11:59:44.554 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (7m) 4s ago 7m 342M 2704M 17.2.0 e1d6a67b021e e29eb74df25a 2024-09-17T11:59:44.554 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (6m) 4s ago 6m 351M 2704M 17.2.0 e1d6a67b021e 5d900eb9165b 2024-09-17T11:59:44.555 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (6m) 4s ago 6m 293M 2704M 17.2.0 e1d6a67b021e fae3e1bae466 2024-09-17T11:59:44.555 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (5m) 4s ago 5m 453M 2704M 17.2.0 e1d6a67b021e 71b9f2cdb810 2024-09-17T11:59:44.555 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (24s) 0s ago 9m 40.1M - 2.51.0 1d3b7f56885b bbafa1646aa4 2024-09-17T11:59:44.705 INFO:teuthology.orchestra.run.smithi045.stdout: 5 92303 10.43 MB/sec execute 105 sec latency 92.305 ms 2024-09-17T11:59:45.025 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:44 smithi045 ceph-mon[23737]: mgrmap e29: smithi165.ykdszc(active, since 99s), standbys: smithi045.jtwyrp 2024-09-17T11:59:45.025 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:44 smithi045 ceph-mon[23737]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:59:45.025 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:44 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:45.026 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:44 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:45.026 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:44 smithi045 ceph-mon[23737]: from='client.14782 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:59:45.193 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T11:59:45.193 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T11:59:45.193 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T11:59:45.193 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:59:45.193 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T11:59:45.193 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T11:59:45.193 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:59:45.193 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T11:59:45.193 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T11:59:45.194 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:59:45.194 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T11:59:45.194 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T11:59:45.194 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T11:59:45.194 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T11:59:45.194 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-17T11:59:45.194 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T11:59:45.194 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T11:59:45.194 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T11:59:45.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:44 smithi165 ceph-mon[25702]: mgrmap e29: smithi165.ykdszc(active, since 99s), standbys: smithi045.jtwyrp 2024-09-17T11:59:45.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:44 smithi165 ceph-mon[25702]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:59:45.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:44 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:45.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:44 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:45.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:44 smithi165 ceph-mon[25702]: from='client.14782 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:59:45.704 INFO:teuthology.orchestra.run.smithi045.stdout: 5 92577 10.35 MB/sec execute 106 sec latency 49.646 ms 2024-09-17T11:59:45.736 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T11:59:45.736 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T11:59:45.736 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T11:59:45.736 INFO:teuthology.orchestra.run.smithi045.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T11:59:45.736 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [ 2024-09-17T11:59:45.737 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr" 2024-09-17T11:59:45.737 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-17T11:59:45.737 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "2/23 daemons upgraded", 2024-09-17T11:59:45.737 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "Currently upgrading mgr daemons", 2024-09-17T11:59:45.737 INFO:teuthology.orchestra.run.smithi045.stdout: "is_paused": false 2024-09-17T11:59:45.737 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T11:59:46.040 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:46 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:46.040 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:46 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:46.040 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:46 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/2630877324' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T11:59:46.040 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:46 smithi045 ceph-mon[23737]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:59:46.040 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:46 smithi045 ceph-mon[23737]: pgmap v53: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.8 MiB/s wr, 370 op/s 2024-09-17T11:59:46.402 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_OK 2024-09-17T11:59:46.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:46 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:46.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:46 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:46.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:46 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/2630877324' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T11:59:46.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:46 smithi165 ceph-mon[25702]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T11:59:46.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:46 smithi165 ceph-mon[25702]: pgmap v53: 97 pgs: 97 active+clean; 327 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.8 MiB/s wr, 370 op/s 2024-09-17T11:59:46.704 INFO:teuthology.orchestra.run.smithi045.stdout: 5 92889 10.33 MB/sec execute 107 sec latency 40.639 ms 2024-09-17T11:59:47.064 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:47 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/2186838675' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T11:59:47.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:47 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/2186838675' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T11:59:47.705 INFO:teuthology.orchestra.run.smithi045.stdout: 5 93236 10.34 MB/sec execute 108 sec latency 44.214 ms 2024-09-17T11:59:48.087 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:48 smithi045 ceph-mon[23737]: pgmap v54: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.7 MiB/s wr, 553 op/s 2024-09-17T11:59:48.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:48 smithi165 ceph-mon[25702]: pgmap v54: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.7 MiB/s wr, 553 op/s 2024-09-17T11:59:48.705 INFO:teuthology.orchestra.run.smithi045.stdout: 5 93666 10.33 MB/sec execute 109 sec latency 39.759 ms 2024-09-17T11:59:49.705 INFO:teuthology.orchestra.run.smithi045.stdout: 5 94078 10.30 MB/sec execute 110 sec latency 37.052 ms 2024-09-17T11:59:49.868 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:49 smithi045 ceph-mon[23737]: pgmap v55: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.0 MiB/s wr, 374 op/s 2024-09-17T11:59:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:49 smithi165 ceph-mon[25702]: pgmap v55: 97 pgs: 97 active+clean; 330 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.0 MiB/s wr, 374 op/s 2024-09-17T11:59:50.705 INFO:teuthology.orchestra.run.smithi045.stdout: 5 94414 10.27 MB/sec execute 111 sec latency 51.349 ms 2024-09-17T11:59:51.705 INFO:teuthology.orchestra.run.smithi045.stdout: 5 94818 10.32 MB/sec execute 112 sec latency 33.421 ms 2024-09-17T11:59:52.412 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:52 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:52.412 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:52 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:52.412 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:52 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:59:52.412 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:52 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:59:52.412 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:52 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:52.413 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:52 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:52.413 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:52 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:59:52.413 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:52 smithi045 ceph-mon[23737]: Upgrade: Need to upgrade myself (mgr.smithi165.ykdszc) 2024-09-17T11:59:52.413 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:52 smithi045 ceph-mon[23737]: Upgrade: Need to upgrade myself (mgr.smithi165.ykdszc) 2024-09-17T11:59:52.413 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:52 smithi045 ceph-mon[23737]: Failing over to other MGR 2024-09-17T11:59:52.413 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:52 smithi045 ceph-mon[23737]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "mgr fail", "who": "smithi165.ykdszc"}]: dispatch 2024-09-17T11:59:52.413 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:52 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "mgr fail", "who": "smithi165.ykdszc"}]: dispatch 2024-09-17T11:59:52.413 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:52 smithi045 ceph-mon[23737]: osdmap e71: 8 total, 8 up, 8 in 2024-09-17T11:59:52.413 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:52 smithi045 ceph-mon[23737]: pgmap v57: 97 pgs: 97 active+clean; 323 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.4 MiB/s wr, 327 op/s 2024-09-17T11:59:52.413 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:52 smithi045 ceph-mon[23737]: Standby manager daemon smithi165.ykdszc started 2024-09-17T11:59:52.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:52 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:52.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:52 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:52.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:52 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T11:59:52.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:52 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T11:59:52.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:52 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:52.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:52 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' 2024-09-17T11:59:52.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:52 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:59:52.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:52 smithi165 ceph-mon[25702]: Upgrade: Need to upgrade myself (mgr.smithi165.ykdszc) 2024-09-17T11:59:52.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:52 smithi165 ceph-mon[25702]: Upgrade: Need to upgrade myself (mgr.smithi165.ykdszc) 2024-09-17T11:59:52.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:52 smithi165 ceph-mon[25702]: Failing over to other MGR 2024-09-17T11:59:52.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:52 smithi165 ceph-mon[25702]: from='mgr.24457 172.21.15.165:0/4038593214' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "mgr fail", "who": "smithi165.ykdszc"}]: dispatch 2024-09-17T11:59:52.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:52 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "mgr fail", "who": "smithi165.ykdszc"}]: dispatch 2024-09-17T11:59:52.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:52 smithi165 ceph-mon[25702]: osdmap e71: 8 total, 8 up, 8 in 2024-09-17T11:59:52.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:52 smithi165 ceph-mon[25702]: pgmap v57: 97 pgs: 97 active+clean; 323 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.4 MiB/s wr, 327 op/s 2024-09-17T11:59:52.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:52 smithi165 ceph-mon[25702]: Standby manager daemon smithi165.ykdszc started 2024-09-17T11:59:52.705 INFO:teuthology.orchestra.run.smithi045.stdout: 5 95354 10.43 MB/sec execute 113 sec latency 33.760 ms 2024-09-17T11:59:53.430 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd='[{"prefix": "mgr fail", "who": "smithi165.ykdszc"}]': finished 2024-09-17T11:59:53.430 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: mgrmap e30: smithi045.jtwyrp(active, starting, since 0.686896s), standbys: smithi165.ykdszc 2024-09-17T11:59:53.430 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi045"}]: dispatch 2024-09-17T11:59:53.430 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi165"}]: dispatch 2024-09-17T11:59:53.431 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi045.olwaas"}]: dispatch 2024-09-17T11:59:53.431 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi165.czpmgd"}]: dispatch 2024-09-17T11:59:53.431 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mgr metadata", "who": "smithi045.jtwyrp", "id": "smithi045.jtwyrp"}]: dispatch 2024-09-17T11:59:53.431 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mgr metadata", "who": "smithi165.ykdszc", "id": "smithi165.ykdszc"}]: dispatch 2024-09-17T11:59:53.431 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T11:59:53.431 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T11:59:53.431 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T11:59:53.431 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T11:59:53.431 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T11:59:53.431 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T11:59:53.431 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T11:59:53.431 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T11:59:53.431 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T11:59:53.431 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T11:59:53.432 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T11:59:53.432 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: Manager daemon smithi045.jtwyrp is now available 2024-09-17T11:59:53.432 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:59:53.432 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T11:59:53.432 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T11:59:53.432 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:59:53.432 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:53 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:59:53.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: from='mgr.24457 ' entity='mgr.smithi165.ykdszc' cmd='[{"prefix": "mgr fail", "who": "smithi165.ykdszc"}]': finished 2024-09-17T11:59:53.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: mgrmap e30: smithi045.jtwyrp(active, starting, since 0.686896s), standbys: smithi165.ykdszc 2024-09-17T11:59:53.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi045"}]: dispatch 2024-09-17T11:59:53.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata", "id": "smithi165"}]: dispatch 2024-09-17T11:59:53.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi045.olwaas"}]: dispatch 2024-09-17T11:59:53.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi165.czpmgd"}]: dispatch 2024-09-17T11:59:53.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mgr metadata", "who": "smithi045.jtwyrp", "id": "smithi045.jtwyrp"}]: dispatch 2024-09-17T11:59:53.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mgr metadata", "who": "smithi165.ykdszc", "id": "smithi165.ykdszc"}]: dispatch 2024-09-17T11:59:53.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T11:59:53.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T11:59:53.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T11:59:53.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T11:59:53.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T11:59:53.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T11:59:53.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T11:59:53.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T11:59:53.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T11:59:53.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T11:59:53.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T11:59:53.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: Manager daemon smithi045.jtwyrp is now available 2024-09-17T11:59:53.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T11:59:53.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T11:59:53.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T11:59:53.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:59:53.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:53 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"}]: dispatch 2024-09-17T11:59:53.705 INFO:teuthology.orchestra.run.smithi045.stdout: 5 95627 10.41 MB/sec execute 114 sec latency 62.795 ms 2024-09-17T11:59:54.236 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:54 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:59:54.236 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:54 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:59:54.236 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:54 smithi045 ceph-mon[23737]: mgrmap e31: smithi045.jtwyrp(active, since 2s), standbys: smithi165.ykdszc 2024-09-17T11:59:54.236 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:54 smithi045 ceph-mon[23737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T11:59:54.236 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:54 smithi045 ceph-mon[23737]: pgmap v3: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-17T11:59:54.236 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:54 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:59:54.236 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:54 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:59:54.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:54 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:59:54.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:54 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"}]: dispatch 2024-09-17T11:59:54.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:54 smithi165 ceph-mon[25702]: mgrmap e31: smithi045.jtwyrp(active, since 2s), standbys: smithi165.ykdszc 2024-09-17T11:59:54.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:54 smithi165 ceph-mon[25702]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T11:59:54.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:54 smithi165 ceph-mon[25702]: pgmap v3: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-17T11:59:54.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:54 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:59:54.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:54 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:59:54.705 INFO:teuthology.orchestra.run.smithi045.stdout: 5 95851 10.36 MB/sec execute 115 sec latency 88.078 ms 2024-09-17T11:59:55.705 INFO:teuthology.orchestra.run.smithi045.stdout: 5 96106 10.28 MB/sec execute 116 sec latency 44.374 ms 2024-09-17T11:59:55.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:55 smithi165 ceph-mon[25702]: pgmap v4: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-17T11:59:55.771 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:55 smithi045 ceph-mon[23737]: pgmap v4: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-17T11:59:56.705 INFO:teuthology.orchestra.run.smithi045.stdout: 5 96378 10.25 MB/sec execute 117 sec latency 61.645 ms 2024-09-17T11:59:56.806 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:56 smithi045 ceph-mon[23737]: [17/Sep/2024:11:59:55] ENGINE Bus STARTING 2024-09-17T11:59:56.807 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:56 smithi045 ceph-mon[23737]: [17/Sep/2024:11:59:55] ENGINE Serving on https://172.21.15.45:7150 2024-09-17T11:59:56.807 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:56 smithi045 ceph-mon[23737]: [17/Sep/2024:11:59:55] ENGINE Error in HTTPServer.serve 2024-09-17T11:59:56.807 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Traceback (most recent call last): 2024-09-17T11:59:56.807 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T11:59:56.807 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: self._connections.run(self.expiration_interval) 2024-09-17T11:59:56.807 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T11:59:56.807 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: self._run(expiration_interval) 2024-09-17T11:59:56.807 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T11:59:56.807 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T11:59:56.807 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T11:59:56.807 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T11:59:56.807 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T11:59:56.807 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: s = self.context.wrap_socket( 2024-09-17T11:59:56.807 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T11:59:56.808 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: return self.sslsocket_class._create( 2024-09-17T11:59:56.808 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T11:59:56.808 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: self.do_handshake() 2024-09-17T11:59:56.808 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T11:59:56.808 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: self._sslobj.do_handshake() 2024-09-17T11:59:56.808 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T11:59:56.808 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:56 smithi045 ceph-mon[23737]: [17/Sep/2024:11:59:55] ENGINE Serving on http://172.21.15.45:8765 2024-09-17T11:59:56.808 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:56 smithi045 ceph-mon[23737]: [17/Sep/2024:11:59:55] ENGINE Bus STARTED 2024-09-17T11:59:56.808 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:56 smithi045 ceph-mon[23737]: mgrmap e32: smithi045.jtwyrp(active, since 4s), standbys: smithi165.ykdszc 2024-09-17T11:59:57.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:56 smithi165 ceph-mon[25702]: [17/Sep/2024:11:59:55] ENGINE Bus STARTING 2024-09-17T11:59:57.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:56 smithi165 ceph-mon[25702]: [17/Sep/2024:11:59:55] ENGINE Serving on https://172.21.15.45:7150 2024-09-17T11:59:57.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:56 smithi165 ceph-mon[25702]: [17/Sep/2024:11:59:55] ENGINE Error in HTTPServer.serve 2024-09-17T11:59:57.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Traceback (most recent call last): 2024-09-17T11:59:57.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T11:59:57.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: self._connections.run(self.expiration_interval) 2024-09-17T11:59:57.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T11:59:57.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: self._run(expiration_interval) 2024-09-17T11:59:57.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T11:59:57.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T11:59:57.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T11:59:57.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T11:59:57.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T11:59:57.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: s = self.context.wrap_socket( 2024-09-17T11:59:57.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T11:59:57.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: return self.sslsocket_class._create( 2024-09-17T11:59:57.018 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T11:59:57.018 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: self.do_handshake() 2024-09-17T11:59:57.018 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T11:59:57.018 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: self._sslobj.do_handshake() 2024-09-17T11:59:57.018 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T11:59:57.018 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:56 smithi165 ceph-mon[25702]: [17/Sep/2024:11:59:55] ENGINE Serving on http://172.21.15.45:8765 2024-09-17T11:59:57.018 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:56 smithi165 ceph-mon[25702]: [17/Sep/2024:11:59:55] ENGINE Bus STARTED 2024-09-17T11:59:57.018 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:56 smithi165 ceph-mon[25702]: mgrmap e32: smithi045.jtwyrp(active, since 4s), standbys: smithi165.ykdszc 2024-09-17T11:59:57.705 INFO:teuthology.orchestra.run.smithi045.stdout: 5 96637 10.22 MB/sec execute 118 sec latency 37.361 ms 2024-09-17T11:59:57.838 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:57 smithi045 ceph-mon[23737]: pgmap v5: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-17T11:59:58.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:57 smithi165 ceph-mon[25702]: pgmap v5: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-17T11:59:58.626 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:58 smithi045 ceph-mon[23737]: mgrmap e33: smithi045.jtwyrp(active, since 6s), standbys: smithi165.ykdszc 2024-09-17T11:59:58.626 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:58 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:59:58.626 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:58 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:59:58.626 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:58 smithi045 ceph-mon[23737]: Standby manager daemon smithi165.ykdszc restarted 2024-09-17T11:59:58.626 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:58 smithi045 ceph-mon[23737]: Standby manager daemon smithi165.ykdszc started 2024-09-17T11:59:58.626 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:58 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.165:0/2521743543' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi165.ykdszc/crt"}]: dispatch 2024-09-17T11:59:58.627 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:58 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.165:0/2521743543' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T11:59:58.627 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:58 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.165:0/2521743543' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi165.ykdszc/key"}]: dispatch 2024-09-17T11:59:58.627 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:58 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.165:0/2521743543' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T11:59:58.627 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:58 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:59:58.627 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:58 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:59:58.705 INFO:teuthology.orchestra.run.smithi045.stdout: 5 97047 10.25 MB/sec execute 119 sec latency 39.827 ms 2024-09-17T11:59:59.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:58 smithi165 ceph-mon[25702]: mgrmap e33: smithi045.jtwyrp(active, since 6s), standbys: smithi165.ykdszc 2024-09-17T11:59:59.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:58 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:59:59.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:58 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:59:59.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:58 smithi165 ceph-mon[25702]: Standby manager daemon smithi165.ykdszc restarted 2024-09-17T11:59:59.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:58 smithi165 ceph-mon[25702]: Standby manager daemon smithi165.ykdszc started 2024-09-17T11:59:59.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:58 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.165:0/2521743543' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi165.ykdszc/crt"}]: dispatch 2024-09-17T11:59:59.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:58 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.165:0/2521743543' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T11:59:59.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:58 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.165:0/2521743543' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi165.ykdszc/key"}]: dispatch 2024-09-17T11:59:59.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:58 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.165:0/2521743543' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T11:59:59.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:58 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:59:59.017 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:58 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T11:59:59.674 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:59 smithi045 ceph-mon[23737]: pgmap v6: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-17T11:59:59.674 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 11:59:59 smithi045 ceph-mon[23737]: mgrmap e34: smithi045.jtwyrp(active, since 7s), standbys: smithi165.ykdszc 2024-09-17T11:59:59.705 INFO:teuthology.orchestra.run.smithi045.stdout: 5 97434 10.22 MB/sec execute 120 sec latency 43.913 ms 2024-09-17T12:00:00.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:59 smithi165 ceph-mon[25702]: pgmap v6: 97 pgs: 97 active+clean; 332 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-17T12:00:00.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 11:59:59 smithi165 ceph-mon[25702]: mgrmap e34: smithi045.jtwyrp(active, since 7s), standbys: smithi165.ykdszc 2024-09-17T12:00:00.690 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:00 smithi045 ceph-mon[23737]: overall HEALTH_OK 2024-09-17T12:00:00.705 INFO:teuthology.orchestra.run.smithi045.stdout: 5 97739 10.17 MB/sec execute 121 sec latency 51.580 ms 2024-09-17T12:00:00.927 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:00 smithi165 ceph-mon[25702]: overall HEALTH_OK 2024-09-17T12:00:01.705 INFO:teuthology.orchestra.run.smithi045.stdout: 5 97925 10.11 MB/sec execute 122 sec latency 72.061 ms 2024-09-17T12:00:01.723 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:01 smithi045 ceph-mon[23737]: pgmap v7: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 3.2 MiB/s wr, 348 op/s 2024-09-17T12:00:02.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:01 smithi165 ceph-mon[25702]: pgmap v7: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 3.2 MiB/s wr, 348 op/s 2024-09-17T12:00:02.705 INFO:teuthology.orchestra.run.smithi045.stdout: 5 98189 10.11 MB/sec execute 123 sec latency 40.151 ms 2024-09-17T12:00:03.514 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:03 smithi045 ceph-mon[23737]: pgmap v8: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 2.4 MiB/s wr, 267 op/s 2024-09-17T12:00:03.514 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:03 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:03.514 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:03 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:03.514 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:03 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:03.514 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:03 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:03.705 INFO:teuthology.orchestra.run.smithi045.stdout: 5 98624 10.18 MB/sec execute 124 sec latency 34.364 ms 2024-09-17T12:00:03.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:03 smithi165 ceph-mon[25702]: pgmap v8: 97 pgs: 97 active+clean; 330 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 2.4 MiB/s wr, 267 op/s 2024-09-17T12:00:03.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:03 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:03.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:03 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:03.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:03 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:03.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:03 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:04.705 INFO:teuthology.orchestra.run.smithi045.stdout: 5 99030 10.22 MB/sec execute 125 sec latency 36.768 ms 2024-09-17T12:00:05.571 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:05 smithi045 ceph-mon[23737]: pgmap v9: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.1 MiB/s wr, 420 op/s 2024-09-17T12:00:05.705 INFO:teuthology.orchestra.run.smithi045.stdout: 5 99342 10.19 MB/sec execute 126 sec latency 43.565 ms 2024-09-17T12:00:05.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:05 smithi165 ceph-mon[25702]: pgmap v9: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.1 MiB/s wr, 420 op/s 2024-09-17T12:00:06.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 99669 10.15 MB/sec execute 127 sec latency 38.812 ms 2024-09-17T12:00:07.453 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:07 smithi045 ceph-mon[23737]: pgmap v10: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 3.6 MiB/s wr, 370 op/s 2024-09-17T12:00:07.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 100047 10.14 MB/sec execute 128 sec latency 39.666 ms 2024-09-17T12:00:07.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:07 smithi165 ceph-mon[25702]: pgmap v10: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 3.6 MiB/s wr, 370 op/s 2024-09-17T12:00:08.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 100455 10.17 MB/sec execute 129 sec latency 43.083 ms 2024-09-17T12:00:08.966 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:08 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T12:00:08.966 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:08 smithi045 ceph-mon[23737]: pgmap v11: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 3.6 MiB/s wr, 370 op/s 2024-09-17T12:00:09.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:08 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T12:00:09.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:08 smithi165 ceph-mon[25702]: pgmap v11: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 3.6 MiB/s wr, 370 op/s 2024-09-17T12:00:09.708 INFO:teuthology.orchestra.run.smithi045.stdout: 5 100896 10.16 MB/sec execute 130 sec latency 41.817 ms 2024-09-17T12:00:10.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 101267 10.11 MB/sec execute 131 sec latency 41.532 ms 2024-09-17T12:00:10.752 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:10 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:10.752 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:10 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:10.752 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:10 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T12:00:10.752 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:10 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T12:00:10.752 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:10 smithi045 ceph-mon[23737]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-17T12:00:10.752 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:10 smithi045 ceph-mon[23737]: Updating smithi165:/etc/ceph/ceph.conf 2024-09-17T12:00:10.753 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:10 smithi045 ceph-mon[23737]: Updating smithi165:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T12:00:10.753 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:10 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:10.753 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:10 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:10.753 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:10 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:10.753 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:10 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:10.753 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:10 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:10.753 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:10 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:10.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:10 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:10.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:10 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:10.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:10 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T12:00:10.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:10 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T12:00:10.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:10 smithi165 ceph-mon[25702]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-17T12:00:10.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:10 smithi165 ceph-mon[25702]: Updating smithi165:/etc/ceph/ceph.conf 2024-09-17T12:00:10.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:10 smithi165 ceph-mon[25702]: Updating smithi165:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T12:00:10.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:10 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:10.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:10 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:10.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:10 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:10.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:10 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:10.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:10 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:10.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:10 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:11.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 101593 10.10 MB/sec execute 132 sec latency 37.283 ms 2024-09-17T12:00:11.760 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:11 smithi045 ceph-mon[23737]: Updating smithi045:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T12:00:11.760 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:11 smithi045 ceph-mon[23737]: Updating smithi165:/etc/ceph/ceph.client.admin.keyring 2024-09-17T12:00:11.760 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:11 smithi045 ceph-mon[23737]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2024-09-17T12:00:11.760 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:11 smithi045 ceph-mon[23737]: Updating smithi165:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T12:00:11.760 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:11 smithi045 ceph-mon[23737]: Updating smithi045:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T12:00:11.760 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:11 smithi045 ceph-mon[23737]: pgmap v12: 97 pgs: 97 active+clean; 340 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.0 MiB/s wr, 598 op/s 2024-09-17T12:00:11.760 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:11 smithi045 ceph-mon[23737]: Reconfiguring prometheus.smithi045 (dependencies changed)... 2024-09-17T12:00:11.760 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:11 smithi045 ceph-mon[23737]: Reconfiguring daemon prometheus.smithi045 on smithi045 2024-09-17T12:00:12.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:11 smithi165 ceph-mon[25702]: Updating smithi045:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T12:00:12.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:11 smithi165 ceph-mon[25702]: Updating smithi165:/etc/ceph/ceph.client.admin.keyring 2024-09-17T12:00:12.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:11 smithi165 ceph-mon[25702]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2024-09-17T12:00:12.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:11 smithi165 ceph-mon[25702]: Updating smithi165:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T12:00:12.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:11 smithi165 ceph-mon[25702]: Updating smithi045:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T12:00:12.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:11 smithi165 ceph-mon[25702]: pgmap v12: 97 pgs: 97 active+clean; 340 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.0 MiB/s wr, 598 op/s 2024-09-17T12:00:12.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:11 smithi165 ceph-mon[25702]: Reconfiguring prometheus.smithi045 (dependencies changed)... 2024-09-17T12:00:12.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:11 smithi165 ceph-mon[25702]: Reconfiguring daemon prometheus.smithi045 on smithi045 2024-09-17T12:00:12.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 102008 10.16 MB/sec execute 133 sec latency 59.624 ms 2024-09-17T12:00:13.531 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:13 smithi045 ceph-mon[23737]: pgmap v13: 97 pgs: 97 active+clean; 340 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.3 MiB/s wr, 407 op/s 2024-09-17T12:00:13.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 102383 10.20 MB/sec execute 134 sec latency 39.167 ms 2024-09-17T12:00:13.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:13 smithi165 ceph-mon[25702]: pgmap v13: 97 pgs: 97 active+clean; 340 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.3 MiB/s wr, 407 op/s 2024-09-17T12:00:14.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 102701 10.20 MB/sec execute 135 sec latency 49.586 ms 2024-09-17T12:00:15.556 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:15 smithi045 ceph-mon[23737]: pgmap v14: 97 pgs: 97 active+clean; 338 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.5 MiB/s wr, 626 op/s 2024-09-17T12:00:15.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 102893 10.15 MB/sec execute 136 sec latency 57.810 ms 2024-09-17T12:00:16.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:15 smithi165 ceph-mon[25702]: pgmap v14: 97 pgs: 97 active+clean; 338 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.5 MiB/s wr, 626 op/s 2024-09-17T12:00:16.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 103128 10.10 MB/sec execute 137 sec latency 111.643 ms 2024-09-17T12:00:17.062 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T12:00:17.602 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:17 smithi045 ceph-mon[23737]: pgmap v15: 97 pgs: 97 active+clean; 338 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 447 op/s 2024-09-17T12:00:17.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 103678 10.11 MB/sec execute 138 sec latency 37.367 ms 2024-09-17T12:00:18.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:17 smithi165 ceph-mon[25702]: pgmap v15: 97 pgs: 97 active+clean; 338 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 447 op/s 2024-09-17T12:00:18.316 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T12:00:18.316 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (98s) 15s ago 10m 21.1M - 0.27.0 11f11916f8cd 2a2a54402d2a 2024-09-17T12:00:18.316 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (10m) 15s ago 10m 6413k - 17.2.0 e1d6a67b021e 8c510943b55e 2024-09-17T12:00:18.316 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (10m) 20s ago 10m 6409k - 17.2.0 e1d6a67b021e 687c24a87b70 2024-09-17T12:00:18.317 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (89s) 15s ago 10m 49.2M - dad864ee21e9 371aaf55379f 2024-09-17T12:00:18.317 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (4m) 15s ago 4m 37.0M - 17.2.0 e1d6a67b021e 6a2a4a998dd1 2024-09-17T12:00:18.317 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (4m) 20s ago 4m 15.1M - 17.2.0 e1d6a67b021e d4c2987b9bac 2024-09-17T12:00:18.317 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:8443,9283,8765 running (43s) 15s ago 12m 490M - 19.3.0-4937-g74984932 6c1fcab85602 ae2f507bd67d 2024-09-17T12:00:18.317 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283 running (2m) 20s ago 10m 159M - 19.3.0-4937-g74984932 6c1fcab85602 96e7a3af0643 2024-09-17T12:00:18.317 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (12m) 15s ago 12m 59.6M 2048M 17.2.0 e1d6a67b021e c4256bdf51b7 2024-09-17T12:00:18.317 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (10m) 20s ago 10m 43.4M 2048M 17.2.0 e1d6a67b021e 9219263d3aa8 2024-09-17T12:00:18.317 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (4m) 15s ago 4m 253M - 4.0 e1d6a67b021e b8fc749ebb71 2024-09-17T12:00:18.318 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (4m) 20s ago 4m 81.8M - 4.0 e1d6a67b021e 771b27764432 2024-09-17T12:00:18.318 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (81s) 15s ago 10m 8380k - 1.7.0 72c9c2088986 989b5738cb69 2024-09-17T12:00:18.318 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (52s) 20s ago 10m 7830k - 1.7.0 72c9c2088986 e1e8ffab3f21 2024-09-17T12:00:18.318 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (9m) 15s ago 9m 398M 1937M 17.2.0 e1d6a67b021e e0ce4d29dc64 2024-09-17T12:00:18.318 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (8m) 15s ago 8m 287M 1937M 17.2.0 e1d6a67b021e da8bda7b47b3 2024-09-17T12:00:18.318 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (8m) 15s ago 8m 247M 1937M 17.2.0 e1d6a67b021e 21fe0d9e5ced 2024-09-17T12:00:18.318 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (7m) 15s ago 7m 366M 1937M 17.2.0 e1d6a67b021e 92a2389488e4 2024-09-17T12:00:18.318 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (7m) 20s ago 7m 354M 2704M 17.2.0 e1d6a67b021e e29eb74df25a 2024-09-17T12:00:18.318 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (7m) 20s ago 7m 379M 2704M 17.2.0 e1d6a67b021e 5d900eb9165b 2024-09-17T12:00:18.318 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (6m) 20s ago 6m 320M 2704M 17.2.0 e1d6a67b021e fae3e1bae466 2024-09-17T12:00:18.319 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (6m) 20s ago 6m 464M 2704M 17.2.0 e1d6a67b021e 71b9f2cdb810 2024-09-17T12:00:18.319 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (58s) 15s ago 9m 40.4M - 2.51.0 1d3b7f56885b bbafa1646aa4 2024-09-17T12:00:18.606 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:18 smithi045 ceph-mon[23737]: from='client.14822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:00:18.606 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:18 smithi045 ceph-mon[23737]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:00:18.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 104391 10.19 MB/sec execute 139 sec latency 21.795 ms 2024-09-17T12:00:19.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:18 smithi165 ceph-mon[25702]: from='client.14822 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:00:19.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:18 smithi165 ceph-mon[25702]: from='client.14826 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:00:19.097 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:00:19.097 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T12:00:19.097 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T12:00:19.098 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:00:19.098 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T12:00:19.098 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:00:19.098 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:00:19.098 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T12:00:19.098 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T12:00:19.098 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:00:19.098 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T12:00:19.098 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T12:00:19.098 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:00:19.098 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T12:00:19.098 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-17T12:00:19.098 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:00:19.098 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T12:00:19.099 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:00:19.611 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:19 smithi045 ceph-mon[23737]: pgmap v16: 97 pgs: 97 active+clean; 338 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 447 op/s 2024-09-17T12:00:19.611 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:19 smithi045 ceph-mon[23737]: from='client.14830 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:00:19.611 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:19 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/1452781311' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T12:00:19.611 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:19 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:19.611 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:19 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:19.611 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:19 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T12:00:19.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 104963 10.16 MB/sec execute 140 sec latency 34.893 ms 2024-09-17T12:00:19.754 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:00:19.755 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T12:00:19.755 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T12:00:19.755 INFO:teuthology.orchestra.run.smithi045.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T12:00:19.755 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [ 2024-09-17T12:00:19.755 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr" 2024-09-17T12:00:19.755 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-17T12:00:19.755 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "2/23 daemons upgraded", 2024-09-17T12:00:19.755 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "", 2024-09-17T12:00:19.755 INFO:teuthology.orchestra.run.smithi045.stdout: "is_paused": false 2024-09-17T12:00:19.755 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:00:20.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:19 smithi165 ceph-mon[25702]: pgmap v16: 97 pgs: 97 active+clean; 338 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.6 MiB/s wr, 447 op/s 2024-09-17T12:00:20.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:19 smithi165 ceph-mon[25702]: from='client.14830 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:00:20.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:19 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/1452781311' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T12:00:20.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:19 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:20.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:19 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:20.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:19 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T12:00:20.485 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_OK 2024-09-17T12:00:20.559 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:20 smithi165 ceph-mon[25702]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T12:00:20.559 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:20 smithi165 ceph-mon[25702]: from='client.24595 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:00:20.559 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:20 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T12:00:20.559 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:20 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/3943470077' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T12:00:20.633 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:20 smithi045 ceph-mon[23737]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T12:00:20.633 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:20 smithi045 ceph-mon[23737]: from='client.24595 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:00:20.633 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:20 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T12:00:20.633 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:20 smithi045 ceph-mon[23737]: from='client.? 172.21.15.45:0/3943470077' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T12:00:20.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 105492 10.23 MB/sec execute 141 sec latency 33.934 ms 2024-09-17T12:00:21.649 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:21 smithi045 ceph-mon[23737]: pgmap v17: 97 pgs: 97 active+clean; 345 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.0 MiB/s wr, 692 op/s 2024-09-17T12:00:21.649 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:21 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi165.ykdszc", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T12:00:21.649 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:21 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi165.ykdszc", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T12:00:21.649 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:21 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T12:00:21.649 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:21 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T12:00:21.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 105905 10.29 MB/sec execute 142 sec latency 41.160 ms 2024-09-17T12:00:21.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:21 smithi165 ceph-mon[25702]: pgmap v17: 97 pgs: 97 active+clean; 345 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.0 MiB/s wr, 692 op/s 2024-09-17T12:00:21.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:21 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi165.ykdszc", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T12:00:21.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:21 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi165.ykdszc", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T12:00:21.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:21 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T12:00:21.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:21 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T12:00:22.656 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:22 smithi045 ceph-mon[23737]: Upgrade: Updating mgr.smithi165.ykdszc 2024-09-17T12:00:22.656 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:22 smithi045 ceph-mon[23737]: Deploying daemon mgr.smithi165.ykdszc on smithi165 2024-09-17T12:00:22.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 106321 10.31 MB/sec execute 143 sec latency 47.286 ms 2024-09-17T12:00:22.967 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:22 smithi165 ceph-mon[25702]: Upgrade: Updating mgr.smithi165.ykdszc 2024-09-17T12:00:22.968 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:22 smithi165 ceph-mon[25702]: Deploying daemon mgr.smithi165.ykdszc on smithi165 2024-09-17T12:00:23.685 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:23 smithi045 ceph-mon[23737]: pgmap v18: 97 pgs: 97 active+clean; 345 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.6 MiB/s wr, 465 op/s 2024-09-17T12:00:23.685 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:23 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T12:00:23.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 106563 10.26 MB/sec execute 144 sec latency 54.281 ms 2024-09-17T12:00:23.916 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:23 smithi165 ceph-mon[25702]: pgmap v18: 97 pgs: 97 active+clean; 345 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.6 MiB/s wr, 465 op/s 2024-09-17T12:00:23.916 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:23 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T12:00:24.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 106768 10.21 MB/sec execute 145 sec latency 64.613 ms 2024-09-17T12:00:25.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 107058 10.19 MB/sec execute 146 sec latency 70.339 ms 2024-09-17T12:00:25.731 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:25 smithi045 ceph-mon[23737]: pgmap v19: 97 pgs: 97 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 721 op/s 2024-09-17T12:00:26.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:25 smithi165 ceph-mon[25702]: pgmap v19: 97 pgs: 97 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.5 MiB/s wr, 721 op/s 2024-09-17T12:00:26.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 107275 10.16 MB/sec execute 147 sec latency 71.160 ms 2024-09-17T12:00:26.745 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:26 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:26.745 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:26 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:26.745 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:26 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T12:00:27.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:26 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:27.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:26 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:27.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:26 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T12:00:27.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 107499 10.15 MB/sec execute 148 sec latency 76.992 ms 2024-09-17T12:00:27.826 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:27 smithi045 ceph-mon[23737]: pgmap v20: 97 pgs: 97 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.3 MiB/s wr, 502 op/s 2024-09-17T12:00:28.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:27 smithi165 ceph-mon[25702]: pgmap v20: 97 pgs: 97 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.3 MiB/s wr, 502 op/s 2024-09-17T12:00:28.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 107797 10.12 MB/sec execute 149 sec latency 62.642 ms 2024-09-17T12:00:28.856 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:28 smithi045 ceph-mon[23737]: pgmap v21: 97 pgs: 97 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.3 MiB/s wr, 502 op/s 2024-09-17T12:00:29.184 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:28 smithi165 ceph-mon[25702]: pgmap v21: 97 pgs: 97 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.3 MiB/s wr, 502 op/s 2024-09-17T12:00:29.707 INFO:teuthology.orchestra.run.smithi045.stdout: 5 108064 10.09 MB/sec execute 150 sec latency 74.941 ms 2024-09-17T12:00:30.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 108455 10.07 MB/sec execute 151 sec latency 54.097 ms 2024-09-17T12:00:30.881 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:30 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:30.881 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:30 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:30.881 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:30 smithi045 ceph-mon[23737]: pgmap v22: 97 pgs: 97 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 669 op/s 2024-09-17T12:00:30.881 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:30 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:30.882 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:30 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:31.138 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:30 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:31.138 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:30 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:31.138 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:30 smithi165 ceph-mon[25702]: pgmap v22: 97 pgs: 97 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 669 op/s 2024-09-17T12:00:31.138 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:30 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:31.138 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:30 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:31.706 INFO:teuthology.orchestra.run.smithi045.stdout: 5 108772 10.06 MB/sec execute 152 sec latency 67.454 ms 2024-09-17T12:00:31.907 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:31 smithi045 ceph-mon[23737]: Standby manager daemon smithi165.ykdszc restarted 2024-09-17T12:00:31.907 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:31 smithi045 ceph-mon[23737]: Standby manager daemon smithi165.ykdszc started 2024-09-17T12:00:31.907 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:31 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.165:0/1470580561' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi165.ykdszc/crt"}]: dispatch 2024-09-17T12:00:31.907 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:31 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.165:0/1470580561' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T12:00:31.907 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:31 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.165:0/1470580561' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi165.ykdszc/key"}]: dispatch 2024-09-17T12:00:31.907 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:31 smithi045 ceph-mon[23737]: from='mgr.? 172.21.15.165:0/1470580561' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T12:00:32.165 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:31 smithi165 ceph-mon[25702]: Standby manager daemon smithi165.ykdszc restarted 2024-09-17T12:00:32.165 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:31 smithi165 ceph-mon[25702]: Standby manager daemon smithi165.ykdszc started 2024-09-17T12:00:32.165 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:31 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.165:0/1470580561' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi165.ykdszc/crt"}]: dispatch 2024-09-17T12:00:32.165 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:31 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.165:0/1470580561' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T12:00:32.166 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:31 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.165:0/1470580561' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi165.ykdszc/key"}]: dispatch 2024-09-17T12:00:32.166 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:31 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.165:0/1470580561' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T12:00:32.707 INFO:teuthology.orchestra.run.smithi045.stdout: 5 109170 10.11 MB/sec execute 153 sec latency 61.790 ms 2024-09-17T12:00:32.953 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:32 smithi045 ceph-mon[23737]: mgrmap e35: smithi045.jtwyrp(active, since 40s), standbys: smithi165.ykdszc 2024-09-17T12:00:32.953 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:32 smithi045 ceph-mon[23737]: pgmap v23: 97 pgs: 97 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.3 MiB/s wr, 424 op/s 2024-09-17T12:00:33.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:32 smithi165 ceph-mon[25702]: mgrmap e35: smithi045.jtwyrp(active, since 40s), standbys: smithi165.ykdszc 2024-09-17T12:00:33.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:32 smithi165 ceph-mon[25702]: pgmap v23: 97 pgs: 97 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.3 MiB/s wr, 424 op/s 2024-09-17T12:00:33.707 INFO:teuthology.orchestra.run.smithi045.stdout: 5 109523 10.12 MB/sec execute 154 sec latency 57.502 ms 2024-09-17T12:00:34.707 INFO:teuthology.orchestra.run.smithi045.stdout: 5 109864 10.12 MB/sec execute 155 sec latency 39.128 ms 2024-09-17T12:00:35.256 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:35 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:35.256 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:35 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:35.256 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:35 smithi045 ceph-mon[23737]: pgmap v24: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 613 op/s 2024-09-17T12:00:35.256 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:35 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:35.256 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:35 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:35.256 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:35 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T12:00:35.257 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:35 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T12:00:35.257 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:35 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:35.257 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:35 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:35.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:35 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:35.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:35 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:35.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:35 smithi165 ceph-mon[25702]: pgmap v24: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 613 op/s 2024-09-17T12:00:35.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:35 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:35.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:35 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:35.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:35 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T12:00:35.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:35 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T12:00:35.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:35 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:35.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:35 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:35.707 INFO:teuthology.orchestra.run.smithi045.stdout: 5 110111 10.09 MB/sec execute 156 sec latency 50.144 ms 2024-09-17T12:00:36.276 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:36 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T12:00:36.276 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:36 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T12:00:36.276 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:36 smithi045 ceph-mon[23737]: Upgrade: Setting container_image for all mgr 2024-09-17T12:00:36.276 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:36 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:36.276 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:36 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi045.jtwyrp"}]: dispatch 2024-09-17T12:00:36.276 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:36 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi045.jtwyrp"}]: dispatch 2024-09-17T12:00:36.277 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:36 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi045.jtwyrp"}]': finished 2024-09-17T12:00:36.277 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:36 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi165.ykdszc"}]: dispatch 2024-09-17T12:00:36.277 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:36 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi165.ykdszc"}]: dispatch 2024-09-17T12:00:36.277 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:36 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi165.ykdszc"}]': finished 2024-09-17T12:00:36.277 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:36 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T12:00:36.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:36 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T12:00:36.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:36 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T12:00:36.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:36 smithi165 ceph-mon[25702]: Upgrade: Setting container_image for all mgr 2024-09-17T12:00:36.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:36 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:36.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:36 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi045.jtwyrp"}]: dispatch 2024-09-17T12:00:36.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:36 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi045.jtwyrp"}]: dispatch 2024-09-17T12:00:36.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:36 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi045.jtwyrp"}]': finished 2024-09-17T12:00:36.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:36 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi165.ykdszc"}]: dispatch 2024-09-17T12:00:36.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:36 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi165.ykdszc"}]: dispatch 2024-09-17T12:00:36.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:36 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi165.ykdszc"}]': finished 2024-09-17T12:00:36.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:36 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T12:00:36.707 INFO:teuthology.orchestra.run.smithi045.stdout: 5 110294 10.04 MB/sec execute 157 sec latency 71.570 ms 2024-09-17T12:00:37.280 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:37 smithi045 ceph-mon[23737]: pgmap v25: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.6 MiB/s wr, 357 op/s 2024-09-17T12:00:37.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:37 smithi165 ceph-mon[25702]: pgmap v25: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.6 MiB/s wr, 357 op/s 2024-09-17T12:00:37.707 INFO:teuthology.orchestra.run.smithi045.stdout: 5 110578 10.01 MB/sec execute 158 sec latency 99.335 ms 2024-09-17T12:00:38.707 INFO:teuthology.orchestra.run.smithi045.stdout: 5 110904 10.00 MB/sec execute 159 sec latency 68.299 ms 2024-09-17T12:00:38.824 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:38 smithi045 ceph-mon[23737]: Upgrade: Updating mon.smithi045 2024-09-17T12:00:38.824 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:38 smithi045 ceph-mon[23737]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:38.824 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:38 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T12:00:38.824 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:38 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T12:00:38.824 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:38 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T12:00:38.824 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:38 smithi045 ceph-mon[23737]: Deploying daemon mon.smithi045 on smithi045 2024-09-17T12:00:38.825 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:38 smithi045 ceph-mon[23737]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T12:00:39.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:38 smithi165 ceph-mon[25702]: Upgrade: Updating mon.smithi045 2024-09-17T12:00:39.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:38 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:39.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:38 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T12:00:39.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:38 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T12:00:39.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:38 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T12:00:39.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:38 smithi165 ceph-mon[25702]: Deploying daemon mon.smithi045 on smithi045 2024-09-17T12:00:39.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:38 smithi165 ceph-mon[25702]: from='mgr.14770 172.21.15.45:0/1837747402' entity='mgr.smithi045.jtwyrp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T12:00:39.707 INFO:teuthology.orchestra.run.smithi045.stdout: 5 111235 10.00 MB/sec execute 160 sec latency 83.425 ms 2024-09-17T12:00:39.791 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:39 smithi045 ceph-mon[23737]: pgmap v26: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.6 MiB/s wr, 357 op/s 2024-09-17T12:00:40.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:39 smithi165 ceph-mon[25702]: pgmap v26: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.6 MiB/s wr, 357 op/s 2024-09-17T12:00:40.298 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:40 smithi045 systemd[1]: Stopping Ceph mon.smithi045 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:00:40.552 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:40 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045[23733]: 2024-09-17T12:00:40.539+0000 7efe90506700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi045 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T12:00:40.552 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:40 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045[23733]: 2024-09-17T12:00:40.539+0000 7efe90506700 -1 mon.smithi045@0(leader) e2 *** Got Signal Terminated *** 2024-09-17T12:00:40.707 INFO:teuthology.orchestra.run.smithi045.stdout: 5 111632 9.99 MB/sec execute 161 sec latency 44.300 ms 2024-09-17T12:00:40.805 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:40 smithi045 podman[63724]: 2024-09-17 12:00:40.703735268 +0000 UTC m=+0.296045335 container died c4256bdf51b7d955c4309544040231c06bc9b4544fcc6131c1d2d633fb0ddccc (image=quay.io/ceph/ceph:v17.2.0, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , distribution-scope=public, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., architecture=x86_64, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, version=8, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, release=754, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-17T12:00:41.558 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:41 smithi045 podman[63724]: 2024-09-17 12:00:41.479314418 +0000 UTC m=+1.071624492 container cleanup c4256bdf51b7d955c4309544040231c06bc9b4544fcc6131c1d2d633fb0ddccc (image=quay.io/ceph/ceph:v17.2.0, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045, RELEASE=HEAD, vcs-type=git, architecture=x86_64, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0) 2024-09-17T12:00:41.558 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:41 smithi045 bash[63724]: ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045 2024-09-17T12:00:41.715 INFO:teuthology.orchestra.run.smithi045.stdout: 5 112037 9.98 MB/sec execute 162 sec latency 39.190 ms 2024-09-17T12:00:41.809 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:41 smithi045 podman[63738]: 2024-09-17 12:00:41.797021863 +0000 UTC m=+1.094845916 container remove c4256bdf51b7d955c4309544040231c06bc9b4544fcc6131c1d2d633fb0ddccc (image=quay.io/ceph/ceph:v17.2.0, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, ceph=True, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, release=754, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, name=centos-stream, GIT_BRANCH=HEAD, architecture=x86_64, RELEASE=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, version=8, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-17T12:00:42.332 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:42 smithi045 systemd[1]: ceph-a12537de-74ea-11ef-bceb-c7b262605968@mon.smithi045.service: Deactivated successfully. 2024-09-17T12:00:42.332 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:42 smithi045 systemd[1]: Stopped Ceph mon.smithi045 for a12537de-74ea-11ef-bceb-c7b262605968. 2024-09-17T12:00:42.332 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:42 smithi045 systemd[1]: ceph-a12537de-74ea-11ef-bceb-c7b262605968@mon.smithi045.service: Consumed 17.148s CPU time. 2024-09-17T12:00:42.583 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:42 smithi045 systemd[1]: Starting Ceph mon.smithi045 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:00:42.715 INFO:teuthology.orchestra.run.smithi045.stdout: 5 112382 9.98 MB/sec execute 163 sec latency 99.557 ms 2024-09-17T12:00:42.837 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:42 smithi045 podman[63861]: 2024-09-17 12:00:42.558500933 +0000 UTC m=+0.019855137 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:00:42.837 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:42 smithi045 podman[63861]: 2024-09-17 12:00:42.734340081 +0000 UTC m=+0.195694280 container create 737eec78e0f09fb20b773507fa81cd666ce003d854ab3b58021f4490c035c057 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=main-7498493, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T12:00:43.354 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 podman[63861]: 2024-09-17 12:00:43.102866461 +0000 UTC m=+0.564220666 container init 737eec78e0f09fb20b773507fa81cd666ce003d854ab3b58021f4490c035c057 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=main-7498493, maintainer=Guillaume Abrioux ) 2024-09-17T12:00:43.355 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 podman[63861]: 2024-09-17 12:00:43.10964871 +0000 UTC m=+0.571002913 container start 737eec78e0f09fb20b773507fa81cd666ce003d854ab3b58021f4490c035c057 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, RELEASE=main-7498493, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-17T12:00:43.355 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: set uid:gid to 167:167 (ceph:ceph) 2024-09-17T12:00:43.355 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev), process ceph-mon, pid 2 2024-09-17T12:00:43.355 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: pidfile_write: ignore empty --pid-file 2024-09-17T12:00:43.355 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: load: jerasure load: lrc 2024-09-17T12:00:43.355 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: RocksDB version: 7.9.2 2024-09-17T12:00:43.355 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Git sha 0 2024-09-17T12:00:43.355 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Compile date 2024-09-15 20:52:36 2024-09-17T12:00:43.355 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: DB SUMMARY 2024-09-17T12:00:43.355 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: DB Session ID: U254PZHU945JFF735J20 2024-09-17T12:00:43.355 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: CURRENT file: CURRENT 2024-09-17T12:00:43.356 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: IDENTITY file: IDENTITY 2024-09-17T12:00:43.356 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: MANIFEST file: MANIFEST-000015 size: 978 Bytes 2024-09-17T12:00:43.356 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi045/store.db dir, Total Num: 1, files: 000030.sst 2024-09-17T12:00:43.356 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi045/store.db: 000028.log size: 2809233 ; 2024-09-17T12:00:43.356 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.error_if_exists: 0 2024-09-17T12:00:43.356 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.create_if_missing: 0 2024-09-17T12:00:43.356 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.paranoid_checks: 1 2024-09-17T12:00:43.356 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-17T12:00:43.356 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-17T12:00:43.356 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-17T12:00:43.356 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.env: 0x55a8e0636120 2024-09-17T12:00:43.356 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.fs: PosixFileSystem 2024-09-17T12:00:43.356 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.info_log: 0x55a8e13c58e0 2024-09-17T12:00:43.356 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_file_opening_threads: 16 2024-09-17T12:00:43.357 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.statistics: (nil) 2024-09-17T12:00:43.357 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.use_fsync: 0 2024-09-17T12:00:43.357 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_log_file_size: 0 2024-09-17T12:00:43.357 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T12:00:43.357 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T12:00:43.357 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.keep_log_file_num: 1000 2024-09-17T12:00:43.358 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.recycle_log_file_num: 0 2024-09-17T12:00:43.358 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.allow_fallocate: 1 2024-09-17T12:00:43.358 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.allow_mmap_reads: 0 2024-09-17T12:00:43.358 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.allow_mmap_writes: 0 2024-09-17T12:00:43.358 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.use_direct_reads: 0 2024-09-17T12:00:43.358 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T12:00:43.358 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.create_missing_column_families: 0 2024-09-17T12:00:43.358 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.db_log_dir: 2024-09-17T12:00:43.358 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.wal_dir: 2024-09-17T12:00:43.358 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T12:00:43.358 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T12:00:43.358 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T12:00:43.358 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T12:00:43.358 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T12:00:43.359 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T12:00:43.359 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.advise_random_on_open: 1 2024-09-17T12:00:43.359 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.db_write_buffer_size: 0 2024-09-17T12:00:43.359 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.write_buffer_manager: 0x55a8e13c9b80 2024-09-17T12:00:43.359 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T12:00:43.359 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T12:00:43.359 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T12:00:43.359 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.rate_limiter: (nil) 2024-09-17T12:00:43.359 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T12:00:43.359 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.wal_recovery_mode: 2 2024-09-17T12:00:43.359 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.enable_thread_tracking: 0 2024-09-17T12:00:43.359 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.enable_pipelined_write: 0 2024-09-17T12:00:43.359 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.unordered_write: 0 2024-09-17T12:00:43.359 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T12:00:43.360 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T12:00:43.360 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T12:00:43.360 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T12:00:43.360 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.row_cache: None 2024-09-17T12:00:43.360 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.wal_filter: None 2024-09-17T12:00:43.360 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T12:00:43.360 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.allow_ingest_behind: 0 2024-09-17T12:00:43.360 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.two_write_queues: 0 2024-09-17T12:00:43.360 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.manual_wal_flush: 0 2024-09-17T12:00:43.360 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.wal_compression: 0 2024-09-17T12:00:43.360 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.atomic_flush: 0 2024-09-17T12:00:43.360 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T12:00:43.360 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T12:00:43.361 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T12:00:43.361 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.log_readahead_size: 0 2024-09-17T12:00:43.361 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-17T12:00:43.361 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.best_efforts_recovery: 0 2024-09-17T12:00:43.361 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-17T12:00:43.361 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-17T12:00:43.361 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.allow_data_in_errors: 0 2024-09-17T12:00:43.361 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.db_host_id: __hostname__ 2024-09-17T12:00:43.361 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-17T12:00:43.361 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_background_jobs: 2 2024-09-17T12:00:43.361 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_background_compactions: -1 2024-09-17T12:00:43.361 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_subcompactions: 1 2024-09-17T12:00:43.361 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T12:00:43.361 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T12:00:43.362 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T12:00:43.362 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_total_wal_size: 0 2024-09-17T12:00:43.362 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T12:00:43.362 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T12:00:43.362 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T12:00:43.362 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T12:00:43.362 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_open_files: -1 2024-09-17T12:00:43.362 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.bytes_per_sync: 0 2024-09-17T12:00:43.362 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T12:00:43.362 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T12:00:43.362 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.compaction_readahead_size: 0 2024-09-17T12:00:43.362 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_background_flushes: -1 2024-09-17T12:00:43.362 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Compression algorithms supported: 2024-09-17T12:00:43.362 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: kZSTD supported: 0 2024-09-17T12:00:43.363 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: kXpressCompression supported: 0 2024-09-17T12:00:43.363 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: kBZip2Compression supported: 0 2024-09-17T12:00:43.363 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T12:00:43.363 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: kLZ4Compression supported: 1 2024-09-17T12:00:43.363 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: kZlibCompression supported: 1 2024-09-17T12:00:43.363 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: kLZ4HCCompression supported: 1 2024-09-17T12:00:43.364 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: kSnappyCompression supported: 1 2024-09-17T12:00:43.364 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T12:00:43.364 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-17T12:00:43.364 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi045/store.db/MANIFEST-000015 2024-09-17T12:00:43.364 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-17T12:00:43.364 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T12:00:43.364 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.merge_operator: 2024-09-17T12:00:43.364 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.compaction_filter: None 2024-09-17T12:00:43.364 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.compaction_filter_factory: None 2024-09-17T12:00:43.364 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.sst_partitioner_factory: None 2024-09-17T12:00:43.365 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T12:00:43.365 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T12:00:43.365 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a8e13c5540) 2024-09-17T12:00:43.365 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: cache_index_and_filter_blocks: 1 2024-09-17T12:00:43.365 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T12:00:43.365 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T12:00:43.365 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: pin_top_level_index_and_filter: 1 2024-09-17T12:00:43.365 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: index_type: 0 2024-09-17T12:00:43.365 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: data_block_index_type: 0 2024-09-17T12:00:43.365 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: index_shortening: 1 2024-09-17T12:00:43.365 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-17T12:00:43.365 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: checksum: 4 2024-09-17T12:00:43.365 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: no_block_cache: 0 2024-09-17T12:00:43.365 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: block_cache: 0x55a8e13b9610 2024-09-17T12:00:43.366 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: block_cache_name: BinnedLRUCache 2024-09-17T12:00:43.366 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: block_cache_options: 2024-09-17T12:00:43.366 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: capacity : 536870912 2024-09-17T12:00:43.366 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: num_shard_bits : 4 2024-09-17T12:00:43.366 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: strict_capacity_limit : 0 2024-09-17T12:00:43.366 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: high_pri_pool_ratio: 0.000 2024-09-17T12:00:43.366 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: block_cache_compressed: (nil) 2024-09-17T12:00:43.366 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: persistent_cache: (nil) 2024-09-17T12:00:43.366 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: block_size: 4096 2024-09-17T12:00:43.366 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: block_size_deviation: 10 2024-09-17T12:00:43.366 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: block_restart_interval: 16 2024-09-17T12:00:43.366 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: index_block_restart_interval: 1 2024-09-17T12:00:43.366 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: metadata_block_size: 4096 2024-09-17T12:00:43.366 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: partition_filters: 0 2024-09-17T12:00:43.367 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: use_delta_encoding: 1 2024-09-17T12:00:43.367 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: filter_policy: bloomfilter 2024-09-17T12:00:43.367 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: whole_key_filtering: 1 2024-09-17T12:00:43.367 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: verify_compression: 0 2024-09-17T12:00:43.367 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: read_amp_bytes_per_bit: 0 2024-09-17T12:00:43.367 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: format_version: 5 2024-09-17T12:00:43.367 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: enable_index_compression: 1 2024-09-17T12:00:43.367 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: block_align: 0 2024-09-17T12:00:43.367 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: max_auto_readahead_size: 262144 2024-09-17T12:00:43.367 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: prepopulate_block_cache: 0 2024-09-17T12:00:43.367 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: initial_auto_readahead_size: 8192 2024-09-17T12:00:43.367 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: num_file_reads_for_auto_readahead: 2 2024-09-17T12:00:43.367 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.write_buffer_size: 33554432 2024-09-17T12:00:43.367 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_write_buffer_number: 2 2024-09-17T12:00:43.367 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.compression: NoCompression 2024-09-17T12:00:43.368 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.bottommost_compression: Disabled 2024-09-17T12:00:43.368 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.prefix_extractor: nullptr 2024-09-17T12:00:43.368 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T12:00:43.368 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.num_levels: 7 2024-09-17T12:00:43.368 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T12:00:43.368 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T12:00:43.368 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T12:00:43.368 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T12:00:43.368 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T12:00:43.368 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T12:00:43.368 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T12:00:43.368 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T12:00:43.368 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-17T12:00:43.369 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T12:00:43.369 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-17T12:00:43.369 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-17T12:00:43.369 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T12:00:43.369 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.compression_opts.level: 32767 2024-09-17T12:00:43.369 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.compression_opts.strategy: 0 2024-09-17T12:00:43.369 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T12:00:43.369 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T12:00:43.369 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-17T12:00:43.369 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-17T12:00:43.369 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.compression_opts.enabled: false 2024-09-17T12:00:43.369 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-17T12:00:43.369 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T12:00:43.369 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T12:00:43.370 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T12:00:43.370 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.target_file_size_base: 67108864 2024-09-17T12:00:43.370 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T12:00:43.370 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T12:00:43.370 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T12:00:43.370 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T12:00:43.370 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T12:00:43.370 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T12:00:43.370 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T12:00:43.370 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T12:00:43.370 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T12:00:43.370 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T12:00:43.370 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T12:00:43.371 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T12:00:43.371 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T12:00:43.371 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-17T12:00:43.371 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.arena_block_size: 1048576 2024-09-17T12:00:43.371 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T12:00:43.371 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T12:00:43.371 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.disable_auto_compactions: 0 2024-09-17T12:00:43.371 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T12:00:43.371 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T12:00:43.371 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T12:00:43.371 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T12:00:43.371 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T12:00:43.371 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T12:00:43.371 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T12:00:43.371 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T12:00:43.372 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T12:00:43.372 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T12:00:43.372 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-17T12:00:43.372 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.inplace_update_support: 0 2024-09-17T12:00:43.372 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T12:00:43.372 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T12:00:43.372 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T12:00:43.372 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T12:00:43.372 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.bloom_locality: 0 2024-09-17T12:00:43.372 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.max_successive_merges: 0 2024-09-17T12:00:43.372 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T12:00:43.372 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.paranoid_file_checks: 0 2024-09-17T12:00:43.372 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.force_consistency_checks: 1 2024-09-17T12:00:43.372 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.report_bg_io_stats: 0 2024-09-17T12:00:43.373 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.ttl: 2592000 2024-09-17T12:00:43.373 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T12:00:43.373 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-17T12:00:43.373 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-17T12:00:43.373 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.enable_blob_files: false 2024-09-17T12:00:43.373 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.min_blob_size: 0 2024-09-17T12:00:43.373 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.blob_file_size: 268435456 2024-09-17T12:00:43.373 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-17T12:00:43.373 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-17T12:00:43.373 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-17T12:00:43.373 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-17T12:00:43.373 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-17T12:00:43.373 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.blob_file_starting_level: 0 2024-09-17T12:00:43.373 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-17T12:00:43.374 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-09-17T12:00:43.374 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T12:00:43.374 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi045/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11848, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T12:00:43.374 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-09-17T12:00:43.374 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 9e4595b0-aa55-453e-bdf4-bc0d9435fd9c 2024-09-17T12:00:43.374 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726574443148260, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-09-17T12:00:43.374 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-09-17T12:00:43.374 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726574443160321, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 2608974, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11846, "largest_seqno": 12871, "table_properties": {"data_size": 2603702, "index_size": 3075, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1157, "raw_key_size": 10399, "raw_average_key_size": 23, "raw_value_size": 2593951, "raw_average_value_size": 5790, "num_data_blocks": 139, "num_entries": 448, "num_filter_entries": 448, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726574443, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "9e4595b0-aa55-453e-bdf4-bc0d9435fd9c", "db_session_id": "U254PZHU945JFF735J20", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-09-17T12:00:43.374 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726574443160501, "job": 1, "event": "recovery_finished"} 2024-09-17T12:00:43.374 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-09-17T12:00:43.375 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T12:00:43.375 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi045/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-17T12:00:43.375 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55a8e13e8e00 2024-09-17T12:00:43.375 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: DB pointer 0x55a8e14ea000 2024-09-17T12:00:43.375 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-17T12:00:43.375 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-17T12:00:43.375 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: ** DB Stats ** 2024-09-17T12:00:43.375 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T12:00:43.376 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T12:00:43.376 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T12:00:43.376 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T12:00:43.376 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T12:00:43.376 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T12:00:43.376 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T12:00:43.376 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: 2024-09-17T12:00:43.376 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: ** Compaction Stats [default] ** 2024-09-17T12:00:43.376 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T12:00:43.376 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-17T12:00:43.376 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: L0 1/0 2.49 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 433.0 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-17T12:00:43.376 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: L6 1/0 6.81 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-17T12:00:43.376 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Sum 2/0 9.30 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 433.0 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-17T12:00:43.377 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 433.0 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-17T12:00:43.377 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: 2024-09-17T12:00:43.377 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: ** Compaction Stats [default] ** 2024-09-17T12:00:43.377 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T12:00:43.377 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T12:00:43.377 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 433.0 0.01 0.00 1 0.006 0 0 0.0 0.0 2024-09-17T12:00:43.377 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: 2024-09-17T12:00:43.377 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-17T12:00:43.377 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: 2024-09-17T12:00:43.377 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T12:00:43.377 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-09-17T12:00:43.377 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T12:00:43.377 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T12:00:43.377 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T12:00:43.378 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T12:00:43.378 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Cumulative compaction: 0.00 GB write, 160.87 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T12:00:43.378 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Interval compaction: 0.00 GB write, 160.87 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T12:00:43.378 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T12:00:43.378 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Block cache BinnedLRUCache@0x55a8e13b9610#2 capacity: 512.00 MB usage: 4.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-17T12:00:43.378 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.28 KB,0.000244379%) IndexBlock(1,3.33 KB,0.000634789%) Misc(1,0.00 KB,0%) 2024-09-17T12:00:43.378 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: 2024-09-17T12:00:43.378 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T12:00:43.378 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: starting mon.smithi045 rank 0 at public addrs [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] at bind addrs [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi045 fsid a12537de-74ea-11ef-bceb-c7b262605968 2024-09-17T12:00:43.378 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: mon.smithi045@-1(???) e2 preinit fsid a12537de-74ea-11ef-bceb-c7b262605968 2024-09-17T12:00:43.378 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: mon.smithi045@-1(???).mds e7 new map 2024-09-17T12:00:43.378 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: mon.smithi045@-1(???).mds e7 print_map 2024-09-17T12:00:43.378 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: e7 2024-09-17T12:00:43.378 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-17T12:00:43.379 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T12:00:43.379 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T12:00:43.379 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: legacy client fscid: 1 2024-09-17T12:00:43.379 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: 2024-09-17T12:00:43.379 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Filesystem 'foofs' (1) 2024-09-17T12:00:43.379 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: fs_name foofs 2024-09-17T12:00:43.379 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: epoch 5 2024-09-17T12:00:43.379 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-17T12:00:43.379 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: created 2024-09-17T11:55:22.360258+0000 2024-09-17T12:00:43.379 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: modified 2024-09-17T11:55:27.378128+0000 2024-09-17T12:00:43.379 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: tableserver 0 2024-09-17T12:00:43.379 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: root 0 2024-09-17T12:00:43.379 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: session_timeout 60 2024-09-17T12:00:43.380 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: session_autoclose 300 2024-09-17T12:00:43.380 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: max_file_size 1099511627776 2024-09-17T12:00:43.380 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: max_xattr_size 65536 2024-09-17T12:00:43.380 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: required_client_features {} 2024-09-17T12:00:43.380 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: last_failure 0 2024-09-17T12:00:43.380 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: last_failure_osd_epoch 0 2024-09-17T12:00:43.380 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T12:00:43.380 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: max_mds 1 2024-09-17T12:00:43.380 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: in 0 2024-09-17T12:00:43.380 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: up {0=14490} 2024-09-17T12:00:43.380 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: failed 2024-09-17T12:00:43.380 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: damaged 2024-09-17T12:00:43.380 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: stopped 2024-09-17T12:00:43.380 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: data_pools [3] 2024-09-17T12:00:43.381 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: metadata_pool 2 2024-09-17T12:00:43.381 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: inline_data disabled 2024-09-17T12:00:43.381 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: balancer 2024-09-17T12:00:43.381 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: bal_rank_mask -1 2024-09-17T12:00:43.381 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: standby_count_wanted 1 2024-09-17T12:00:43.381 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: qdb_cluster leader: 0 members: 2024-09-17T12:00:43.381 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: [mds.foofs.smithi045.olwaas{0:14490} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.45:6834/558522566,v1:172.21.15.45:6835/558522566] compat {c=[1],r=[1],i=[7ff]}] 2024-09-17T12:00:43.381 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: 2024-09-17T12:00:43.381 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: 2024-09-17T12:00:43.381 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Standby daemons: 2024-09-17T12:00:43.381 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: 2024-09-17T12:00:43.381 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: [mds.foofs.smithi165.czpmgd{-1:24315} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.165:6832/1936605703,v1:172.21.15.165:6833/1936605703] compat {c=[1],r=[1],i=[7ff]}] 2024-09-17T12:00:43.381 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: mon.smithi045@-1(???).osd e71 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-17T12:00:43.382 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: mon.smithi045@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T12:00:43.382 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: mon.smithi045@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T12:00:43.382 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: mon.smithi045@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T12:00:43.382 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 ceph-mon[63880]: mon.smithi045@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-09-17T12:00:43.382 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 bash[63861]: 737eec78e0f09fb20b773507fa81cd666ce003d854ab3b58021f4490c035c057 2024-09-17T12:00:43.382 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:43 smithi045 systemd[1]: Started Ceph mon.smithi045 for a12537de-74ea-11ef-bceb-c7b262605968. 2024-09-17T12:00:43.715 INFO:teuthology.orchestra.run.smithi045.stdout: 5 112785 10.02 MB/sec execute 164 sec latency 39.495 ms 2024-09-17T12:00:44.126 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:44 smithi045 ceph-mon[63880]: pgmap v27: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.4 MiB/s wr, 541 op/s 2024-09-17T12:00:44.126 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:44 smithi045 ceph-mon[63880]: pgmap v28: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.9 MiB/s wr, 374 op/s 2024-09-17T12:00:44.126 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:44 smithi045 ceph-mon[63880]: mon.smithi045 calling monitor election 2024-09-17T12:00:44.126 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:44 smithi045 ceph-mon[63880]: mon.smithi045 is new leader, mons smithi045,smithi165 in quorum (ranks 0,1) 2024-09-17T12:00:44.126 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:44 smithi045 ceph-mon[63880]: monmap epoch 2 2024-09-17T12:00:44.127 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:44 smithi045 ceph-mon[63880]: fsid a12537de-74ea-11ef-bceb-c7b262605968 2024-09-17T12:00:44.127 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:44 smithi045 ceph-mon[63880]: last_changed 2024-09-17T11:50:08.996728+0000 2024-09-17T12:00:44.127 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:44 smithi045 ceph-mon[63880]: created 2024-09-17T11:48:00.407559+0000 2024-09-17T12:00:44.127 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:44 smithi045 ceph-mon[63880]: min_mon_release 17 (quincy) 2024-09-17T12:00:44.127 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:44 smithi045 ceph-mon[63880]: election_strategy: 1 2024-09-17T12:00:44.127 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:44 smithi045 ceph-mon[63880]: 0: [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] mon.smithi045 2024-09-17T12:00:44.127 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:44 smithi045 ceph-mon[63880]: 1: [v2:172.21.15.165:3300/0,v1:172.21.15.165:6789/0] mon.smithi165 2024-09-17T12:00:44.127 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:44 smithi045 ceph-mon[63880]: fsmap foofs:1 {0=foofs.smithi045.olwaas=up:active} 1 up:standby 2024-09-17T12:00:44.127 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:44 smithi045 ceph-mon[63880]: osdmap e71: 8 total, 8 up, 8 in 2024-09-17T12:00:44.127 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:44 smithi045 ceph-mon[63880]: mgrmap e35: smithi045.jtwyrp(active, since 52s), standbys: smithi165.ykdszc 2024-09-17T12:00:44.127 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:44 smithi045 ceph-mon[63880]: overall HEALTH_OK 2024-09-17T12:00:44.127 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:44 smithi045 ceph-mon[63880]: from='mgr.14770 ' entity='' 2024-09-17T12:00:44.127 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:44 smithi045 ceph-mon[63880]: mgrmap e36: smithi045.jtwyrp(active, since 52s), standbys: smithi165.ykdszc 2024-09-17T12:00:44.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:44 smithi165 ceph-mon[25702]: pgmap v27: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.4 MiB/s wr, 541 op/s 2024-09-17T12:00:44.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:44 smithi165 ceph-mon[25702]: pgmap v28: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.9 MiB/s wr, 374 op/s 2024-09-17T12:00:44.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:44 smithi165 ceph-mon[25702]: mon.smithi045 calling monitor election 2024-09-17T12:00:44.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:44 smithi165 ceph-mon[25702]: mon.smithi045 is new leader, mons smithi045,smithi165 in quorum (ranks 0,1) 2024-09-17T12:00:44.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:44 smithi165 ceph-mon[25702]: monmap epoch 2 2024-09-17T12:00:44.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:44 smithi165 ceph-mon[25702]: fsid a12537de-74ea-11ef-bceb-c7b262605968 2024-09-17T12:00:44.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:44 smithi165 ceph-mon[25702]: last_changed 2024-09-17T11:50:08.996728+0000 2024-09-17T12:00:44.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:44 smithi165 ceph-mon[25702]: created 2024-09-17T11:48:00.407559+0000 2024-09-17T12:00:44.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:44 smithi165 ceph-mon[25702]: min_mon_release 17 (quincy) 2024-09-17T12:00:44.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:44 smithi165 ceph-mon[25702]: election_strategy: 1 2024-09-17T12:00:44.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:44 smithi165 ceph-mon[25702]: 0: [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] mon.smithi045 2024-09-17T12:00:44.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:44 smithi165 ceph-mon[25702]: 1: [v2:172.21.15.165:3300/0,v1:172.21.15.165:6789/0] mon.smithi165 2024-09-17T12:00:44.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:44 smithi165 ceph-mon[25702]: fsmap foofs:1 {0=foofs.smithi045.olwaas=up:active} 1 up:standby 2024-09-17T12:00:44.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:44 smithi165 ceph-mon[25702]: osdmap e71: 8 total, 8 up, 8 in 2024-09-17T12:00:44.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:44 smithi165 ceph-mon[25702]: mgrmap e35: smithi045.jtwyrp(active, since 52s), standbys: smithi165.ykdszc 2024-09-17T12:00:44.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:44 smithi165 ceph-mon[25702]: overall HEALTH_OK 2024-09-17T12:00:44.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:44 smithi165 ceph-mon[25702]: from='mgr.14770 ' entity='' 2024-09-17T12:00:44.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:44 smithi165 ceph-mon[25702]: mgrmap e36: smithi045.jtwyrp(active, since 52s), standbys: smithi165.ykdszc 2024-09-17T12:00:44.715 INFO:teuthology.orchestra.run.smithi045.stdout: 5 113107 10.03 MB/sec execute 165 sec latency 59.224 ms 2024-09-17T12:00:45.715 INFO:teuthology.orchestra.run.smithi045.stdout: 5 113411 10.03 MB/sec execute 166 sec latency 77.588 ms 2024-09-17T12:00:46.144 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:46 smithi045 ceph-mon[63880]: mgrmap e37: smithi045.jtwyrp(active, since 53s), standbys: smithi165.ykdszc 2024-09-17T12:00:46.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:46 smithi165 ceph-mon[25702]: mgrmap e37: smithi045.jtwyrp(active, since 53s), standbys: smithi165.ykdszc 2024-09-17T12:00:46.715 INFO:teuthology.orchestra.run.smithi045.stdout: 5 113664 10.00 MB/sec execute 167 sec latency 66.723 ms 2024-09-17T12:00:47.716 INFO:teuthology.orchestra.run.smithi045.stdout: 5 113830 9.95 MB/sec execute 168 sec latency 75.990 ms 2024-09-17T12:00:48.716 INFO:teuthology.orchestra.run.smithi045.stdout: 5 113999 9.92 MB/sec execute 169 sec latency 78.387 ms 2024-09-17T12:00:49.716 INFO:teuthology.orchestra.run.smithi045.stdout: 5 114212 9.90 MB/sec execute 170 sec latency 66.762 ms 2024-09-17T12:00:50.029 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:49 smithi045 ceph-mon[63880]: Standby manager daemon smithi165.ykdszc restarted 2024-09-17T12:00:50.029 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:49 smithi045 ceph-mon[63880]: Standby manager daemon smithi165.ykdszc started 2024-09-17T12:00:50.029 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:49 smithi045 ceph-mon[63880]: from='mgr.? 172.21.15.165:0/4091120205' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi165.ykdszc/crt"}]: dispatch 2024-09-17T12:00:50.029 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:49 smithi045 ceph-mon[63880]: from='mgr.? 172.21.15.165:0/4091120205' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T12:00:50.029 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:49 smithi045 ceph-mon[63880]: from='mgr.? 172.21.15.165:0/4091120205' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi165.ykdszc/key"}]: dispatch 2024-09-17T12:00:50.029 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:49 smithi045 ceph-mon[63880]: from='mgr.? 172.21.15.165:0/4091120205' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T12:00:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:49 smithi165 ceph-mon[25702]: Standby manager daemon smithi165.ykdszc restarted 2024-09-17T12:00:50.301 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:49 smithi165 ceph-mon[25702]: Standby manager daemon smithi165.ykdszc started 2024-09-17T12:00:50.301 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:49 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.165:0/4091120205' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi165.ykdszc/crt"}]: dispatch 2024-09-17T12:00:50.302 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:49 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.165:0/4091120205' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T12:00:50.302 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:49 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.165:0/4091120205' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi165.ykdszc/key"}]: dispatch 2024-09-17T12:00:50.302 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:49 smithi165 ceph-mon[25702]: from='mgr.? 172.21.15.165:0/4091120205' entity='mgr.smithi165.ykdszc' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T12:00:50.716 INFO:teuthology.orchestra.run.smithi045.stdout: 5 114431 9.86 MB/sec execute 171 sec latency 54.984 ms 2024-09-17T12:00:51.036 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:50 smithi045 ceph-mon[63880]: mgrmap e38: smithi045.jtwyrp(active, since 58s), standbys: smithi165.ykdszc 2024-09-17T12:00:51.036 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:50 smithi045 ceph-mon[63880]: Active manager daemon smithi045.jtwyrp restarted 2024-09-17T12:00:51.036 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:50 smithi045 ceph-mon[63880]: Activating manager daemon smithi045.jtwyrp 2024-09-17T12:00:51.037 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:50 smithi045 ceph-mon[63880]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T12:00:51.037 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:50 smithi045 ceph-mon[63880]: mgrmap e39: smithi045.jtwyrp(active, starting, since 0.0721538s), standbys: smithi165.ykdszc 2024-09-17T12:00:51.037 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mon metadata", "id": "smithi045"} : dispatch 2024-09-17T12:00:51.037 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mon metadata", "id": "smithi165"} : dispatch 2024-09-17T12:00:51.037 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mds metadata", "who": "foofs.smithi045.olwaas"} : dispatch 2024-09-17T12:00:51.037 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mds metadata", "who": "foofs.smithi165.czpmgd"} : dispatch 2024-09-17T12:00:51.037 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mgr metadata", "who": "smithi045.jtwyrp", "id": "smithi045.jtwyrp"} : dispatch 2024-09-17T12:00:51.037 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mgr metadata", "who": "smithi165.ykdszc", "id": "smithi165.ykdszc"} : dispatch 2024-09-17T12:00:51.037 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-17T12:00:51.037 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-17T12:00:51.037 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-17T12:00:51.037 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-17T12:00:51.037 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-17T12:00:51.038 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-17T12:00:51.038 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-17T12:00:51.038 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-17T12:00:51.038 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mds metadata"} : dispatch 2024-09-17T12:00:51.038 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata"} : dispatch 2024-09-17T12:00:51.038 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mon metadata"} : dispatch 2024-09-17T12:00:51.038 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:50 smithi045 ceph-mon[63880]: Manager daemon smithi045.jtwyrp is now available 2024-09-17T12:00:51.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:50 smithi165 ceph-mon[25702]: mgrmap e38: smithi045.jtwyrp(active, since 58s), standbys: smithi165.ykdszc 2024-09-17T12:00:51.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:50 smithi165 ceph-mon[25702]: Active manager daemon smithi045.jtwyrp restarted 2024-09-17T12:00:51.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:50 smithi165 ceph-mon[25702]: Activating manager daemon smithi045.jtwyrp 2024-09-17T12:00:51.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:50 smithi165 ceph-mon[25702]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T12:00:51.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:50 smithi165 ceph-mon[25702]: mgrmap e39: smithi045.jtwyrp(active, starting, since 0.0721538s), standbys: smithi165.ykdszc 2024-09-17T12:00:51.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:50 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mon metadata", "id": "smithi045"} : dispatch 2024-09-17T12:00:51.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:50 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mon metadata", "id": "smithi165"} : dispatch 2024-09-17T12:00:51.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:50 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mds metadata", "who": "foofs.smithi045.olwaas"} : dispatch 2024-09-17T12:00:51.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:50 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mds metadata", "who": "foofs.smithi165.czpmgd"} : dispatch 2024-09-17T12:00:51.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:50 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mgr metadata", "who": "smithi045.jtwyrp", "id": "smithi045.jtwyrp"} : dispatch 2024-09-17T12:00:51.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:50 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mgr metadata", "who": "smithi165.ykdszc", "id": "smithi165.ykdszc"} : dispatch 2024-09-17T12:00:51.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:50 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-17T12:00:51.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:50 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-17T12:00:51.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:50 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-17T12:00:51.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:50 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-17T12:00:51.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:50 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-17T12:00:51.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:50 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-17T12:00:51.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:50 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-17T12:00:51.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:50 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-17T12:00:51.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:50 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mds metadata"} : dispatch 2024-09-17T12:00:51.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:50 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata"} : dispatch 2024-09-17T12:00:51.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:50 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mon metadata"} : dispatch 2024-09-17T12:00:51.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:50 smithi165 ceph-mon[25702]: Manager daemon smithi045.jtwyrp is now available 2024-09-17T12:00:51.716 INFO:teuthology.orchestra.run.smithi045.stdout: 5 114705 9.85 MB/sec execute 172 sec latency 69.423 ms 2024-09-17T12:00:51.919 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T12:00:52.009 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:51 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:00:52.010 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:51 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-17T12:00:52.010 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:51 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:00:52.010 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:51 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"} : dispatch 2024-09-17T12:00:52.010 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:51 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"} : dispatch 2024-09-17T12:00:52.010 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:51 smithi165 ceph-mon[25702]: mgrmap e40: smithi045.jtwyrp(active, since 1.08731s), standbys: smithi165.ykdszc 2024-09-17T12:00:52.060 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:00:52.060 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-17T12:00:52.060 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:00:52.060 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/mirror_snapshot_schedule"} : dispatch 2024-09-17T12:00:52.060 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi045.jtwyrp/trash_purge_schedule"} : dispatch 2024-09-17T12:00:52.060 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:51 smithi045 ceph-mon[63880]: mgrmap e40: smithi045.jtwyrp(active, since 1.08731s), standbys: smithi165.ykdszc 2024-09-17T12:00:52.716 INFO:teuthology.orchestra.run.smithi045.stdout: 5 115140 9.86 MB/sec execute 173 sec latency 51.768 ms 2024-09-17T12:00:53.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:52 smithi165 ceph-mon[25702]: from='client.34106 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:00:53.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:52 smithi165 ceph-mon[25702]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T12:00:53.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:52 smithi165 ceph-mon[25702]: pgmap v3: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-17T12:00:53.063 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:52 smithi045 ceph-mon[63880]: from='client.34106 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:00:53.063 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:52 smithi045 ceph-mon[63880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-17T12:00:53.063 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:52 smithi045 ceph-mon[63880]: pgmap v3: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-17T12:00:53.181 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T12:00:53.181 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (2m) 18s ago 11m 21.1M - 0.27.0 11f11916f8cd 2a2a54402d2a 2024-09-17T12:00:53.181 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (11m) 18s ago 11m 6413k - 17.2.0 e1d6a67b021e 8c510943b55e 2024-09-17T12:00:53.181 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (10m) 23s ago 10m 6409k - 17.2.0 e1d6a67b021e 687c24a87b70 2024-09-17T12:00:53.181 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (2m) 18s ago 11m 55.2M - dad864ee21e9 371aaf55379f 2024-09-17T12:00:53.181 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (5m) 18s ago 5m 37.2M - 17.2.0 e1d6a67b021e 6a2a4a998dd1 2024-09-17T12:00:53.182 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (5m) 23s ago 5m 15.3M - 17.2.0 e1d6a67b021e d4c2987b9bac 2024-09-17T12:00:53.182 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:8443,9283,8765 running (78s) 18s ago 12m 507M - 19.3.0-4937-g74984932 6c1fcab85602 ae2f507bd67d 2024-09-17T12:00:53.182 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283,8765 running (28s) 23s ago 10m 66.8M - 19.3.0-4937-g74984932 6c1fcab85602 53403866652e 2024-09-17T12:00:53.182 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (12m) 18s ago 12m 58.9M 2048M 17.2.0 e1d6a67b021e c4256bdf51b7 2024-09-17T12:00:53.182 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (10m) 23s ago 10m 44.3M 2048M 17.2.0 e1d6a67b021e 9219263d3aa8 2024-09-17T12:00:53.182 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (5m) 18s ago 5m 258M - 4.0 e1d6a67b021e b8fc749ebb71 2024-09-17T12:00:53.182 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (5m) 23s ago 5m 83.4M - 4.0 e1d6a67b021e 771b27764432 2024-09-17T12:00:53.182 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (116s) 18s ago 11m 8266k - 1.7.0 72c9c2088986 989b5738cb69 2024-09-17T12:00:53.182 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (87s) 23s ago 10m 12.0M - 1.7.0 72c9c2088986 e1e8ffab3f21 2024-09-17T12:00:53.182 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (9m) 18s ago 9m 442M 1937M 17.2.0 e1d6a67b021e e0ce4d29dc64 2024-09-17T12:00:53.182 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (9m) 18s ago 9m 323M 1937M 17.2.0 e1d6a67b021e da8bda7b47b3 2024-09-17T12:00:53.182 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (9m) 18s ago 9m 275M 1937M 17.2.0 e1d6a67b021e 21fe0d9e5ced 2024-09-17T12:00:53.182 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (8m) 18s ago 8m 401M 1937M 17.2.0 e1d6a67b021e 92a2389488e4 2024-09-17T12:00:53.183 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (8m) 23s ago 8m 382M 2704M 17.2.0 e1d6a67b021e e29eb74df25a 2024-09-17T12:00:53.183 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (7m) 23s ago 7m 412M 2704M 17.2.0 e1d6a67b021e 5d900eb9165b 2024-09-17T12:00:53.183 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (7m) 23s ago 7m 350M 2704M 17.2.0 e1d6a67b021e fae3e1bae466 2024-09-17T12:00:53.183 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (6m) 23s ago 6m 501M 2704M 17.2.0 e1d6a67b021e 71b9f2cdb810 2024-09-17T12:00:53.183 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (34s) 18s ago 10m 45.9M - 2.51.0 1d3b7f56885b 61b064ad6609 2024-09-17T12:00:53.716 INFO:teuthology.orchestra.run.smithi045.stdout: 5 115784 9.88 MB/sec execute 174 sec latency 37.362 ms 2024-09-17T12:00:53.952 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:00:53.952 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T12:00:53.952 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T12:00:53.952 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 1 2024-09-17T12:00:53.952 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:00:53.952 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T12:00:53.952 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:00:53.953 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:00:53.953 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T12:00:53.953 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T12:00:53.953 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:00:53.953 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T12:00:53.953 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T12:00:53.953 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:00:53.953 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T12:00:53.953 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-09-17T12:00:53.953 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 3 2024-09-17T12:00:53.953 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T12:00:53.953 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:00:54.072 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:53 smithi045 ceph-mon[63880]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:00:54.072 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:53 smithi045 ceph-mon[63880]: pgmap v4: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-17T12:00:54.072 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:53 smithi045 ceph-mon[63880]: mgrmap e41: smithi045.jtwyrp(active, since 2s), standbys: smithi165.ykdszc 2024-09-17T12:00:54.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:53 smithi165 ceph-mon[25702]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:00:54.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:53 smithi165 ceph-mon[25702]: pgmap v4: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-17T12:00:54.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:53 smithi165 ceph-mon[25702]: mgrmap e41: smithi045.jtwyrp(active, since 2s), standbys: smithi165.ykdszc 2024-09-17T12:00:54.573 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:00:54.573 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T12:00:54.574 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T12:00:54.574 INFO:teuthology.orchestra.run.smithi045.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T12:00:54.574 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [ 2024-09-17T12:00:54.574 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr" 2024-09-17T12:00:54.574 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-17T12:00:54.574 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "2/23 daemons upgraded", 2024-09-17T12:00:54.574 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "", 2024-09-17T12:00:54.574 INFO:teuthology.orchestra.run.smithi045.stdout: "is_paused": false 2024-09-17T12:00:54.574 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:00:54.716 INFO:teuthology.orchestra.run.smithi045.stdout: 5 116417 9.97 MB/sec execute 175 sec latency 39.454 ms 2024-09-17T12:00:54.913 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:54 smithi165 ceph-mon[25702]: [17/Sep/2024:12:00:53] ENGINE Bus STARTING 2024-09-17T12:00:54.913 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:54 smithi165 ceph-mon[25702]: [17/Sep/2024:12:00:53] ENGINE Serving on https://172.21.15.45:7150 2024-09-17T12:00:54.913 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:54 smithi165 ceph-mon[25702]: [17/Sep/2024:12:00:53] ENGINE Error in HTTPServer.serve 2024-09-17T12:00:54.913 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Traceback (most recent call last): 2024-09-17T12:00:54.913 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T12:00:54.913 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: self._connections.run(self.expiration_interval) 2024-09-17T12:00:54.914 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T12:00:54.914 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: self._run(expiration_interval) 2024-09-17T12:00:54.914 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T12:00:54.914 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T12:00:54.914 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T12:00:54.914 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T12:00:54.914 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T12:00:54.914 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: s = self.context.wrap_socket( 2024-09-17T12:00:54.914 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T12:00:54.914 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: return self.sslsocket_class._create( 2024-09-17T12:00:54.914 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T12:00:54.914 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: self.do_handshake() 2024-09-17T12:00:54.915 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T12:00:54.915 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: self._sslobj.do_handshake() 2024-09-17T12:00:54.915 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T12:00:54.915 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:54 smithi165 ceph-mon[25702]: [17/Sep/2024:12:00:53] ENGINE Serving on http://172.21.15.45:8765 2024-09-17T12:00:54.915 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:54 smithi165 ceph-mon[25702]: [17/Sep/2024:12:00:53] ENGINE Bus STARTED 2024-09-17T12:00:54.915 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:54 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/2126859674' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T12:00:55.107 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:54 smithi045 ceph-mon[63880]: [17/Sep/2024:12:00:53] ENGINE Bus STARTING 2024-09-17T12:00:55.107 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:54 smithi045 ceph-mon[63880]: [17/Sep/2024:12:00:53] ENGINE Serving on https://172.21.15.45:7150 2024-09-17T12:00:55.107 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:54 smithi045 ceph-mon[63880]: [17/Sep/2024:12:00:53] ENGINE Error in HTTPServer.serve 2024-09-17T12:00:55.107 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: Traceback (most recent call last): 2024-09-17T12:00:55.107 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T12:00:55.107 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: self._connections.run(self.expiration_interval) 2024-09-17T12:00:55.108 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T12:00:55.108 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: self._run(expiration_interval) 2024-09-17T12:00:55.108 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T12:00:55.108 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T12:00:55.108 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T12:00:55.108 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T12:00:55.108 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T12:00:55.108 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: s = self.context.wrap_socket( 2024-09-17T12:00:55.108 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T12:00:55.108 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: return self.sslsocket_class._create( 2024-09-17T12:00:55.108 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T12:00:55.108 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: self.do_handshake() 2024-09-17T12:00:55.109 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T12:00:55.109 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: self._sslobj.do_handshake() 2024-09-17T12:00:55.109 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T12:00:55.109 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:54 smithi045 ceph-mon[63880]: [17/Sep/2024:12:00:53] ENGINE Serving on http://172.21.15.45:8765 2024-09-17T12:00:55.109 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:54 smithi045 ceph-mon[63880]: [17/Sep/2024:12:00:53] ENGINE Bus STARTED 2024-09-17T12:00:55.109 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:54 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/2126859674' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T12:00:55.287 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_OK 2024-09-17T12:00:55.716 INFO:teuthology.orchestra.run.smithi045.stdout: 5 116876 9.96 MB/sec execute 176 sec latency 53.689 ms 2024-09-17T12:00:56.125 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:55 smithi045 ceph-mon[63880]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:00:56.125 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:55 smithi045 ceph-mon[63880]: pgmap v5: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-17T12:00:56.125 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:55 smithi045 ceph-mon[63880]: mgrmap e42: smithi045.jtwyrp(active, since 4s), standbys: smithi165.ykdszc 2024-09-17T12:00:56.125 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:55 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/1939241651' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T12:00:56.125 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:55 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:56.125 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:55 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:56.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:55 smithi165 ceph-mon[25702]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:00:56.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:55 smithi165 ceph-mon[25702]: pgmap v5: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-17T12:00:56.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:55 smithi165 ceph-mon[25702]: mgrmap e42: smithi045.jtwyrp(active, since 4s), standbys: smithi165.ykdszc 2024-09-17T12:00:56.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:55 smithi165 ceph-mon[25702]: from='client.? 172.21.15.45:0/1939241651' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T12:00:56.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:55 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:56.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:55 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:56.716 INFO:teuthology.orchestra.run.smithi045.stdout: 5 117153 9.96 MB/sec execute 177 sec latency 62.059 ms 2024-09-17T12:00:57.660 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:57 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:57.660 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:57 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:57.660 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:57 smithi045 ceph-mon[63880]: pgmap v6: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-17T12:00:57.716 INFO:teuthology.orchestra.run.smithi045.stdout: 5 117431 9.93 MB/sec execute 178 sec latency 63.324 ms 2024-09-17T12:00:58.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:57 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:58.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:57 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:00:58.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:57 smithi165 ceph-mon[25702]: pgmap v6: 97 pgs: 97 active+clean; 331 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-17T12:00:58.716 INFO:teuthology.orchestra.run.smithi045.stdout: 5 117859 9.94 MB/sec execute 179 sec latency 41.683 ms 2024-09-17T12:00:59.716 INFO:teuthology.orchestra.run.smithi045.stdout: 5 118340 9.95 MB/sec execute 180 sec latency 49.249 ms 2024-09-17T12:00:59.945 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:00:59 smithi045 ceph-mon[63880]: pgmap v7: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 31 KiB/s rd, 5.2 MiB/s wr, 482 op/s 2024-09-17T12:01:00.062 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:00:59 smithi165 ceph-mon[25702]: pgmap v7: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 31 KiB/s rd, 5.2 MiB/s wr, 482 op/s 2024-09-17T12:01:00.716 INFO:teuthology.orchestra.run.smithi045.stdout: 5 118982 9.99 MB/sec execute 181 sec latency 31.873 ms 2024-09-17T12:01:01.716 INFO:teuthology.orchestra.run.smithi045.stdout: 5 119607 10.01 MB/sec execute 182 sec latency 40.469 ms 2024-09-17T12:01:01.731 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:01 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:01.731 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:01 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:01.731 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:01 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:01.731 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:01 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:01.731 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:01 smithi045 ceph-mon[63880]: pgmap v8: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 4.0 MiB/s wr, 374 op/s 2024-09-17T12:01:01.731 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:01 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:01.731 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:01 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:01.731 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:01 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:01.731 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:01 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:01:01.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:01 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:01.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:01 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:01.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:01 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:01.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:01 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:01.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:01 smithi165 ceph-mon[25702]: pgmap v8: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 4.0 MiB/s wr, 374 op/s 2024-09-17T12:01:01.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:01 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:01.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:01 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:01.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:01 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:01.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:01 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:01:02.493 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:02 smithi045 ceph-mon[63880]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-17T12:01:02.493 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:02 smithi045 ceph-mon[63880]: Updating smithi165:/etc/ceph/ceph.conf 2024-09-17T12:01:02.493 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:02 smithi045 ceph-mon[63880]: Updating smithi045:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T12:01:02.493 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:02 smithi045 ceph-mon[63880]: Updating smithi165:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T12:01:02.493 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:02 smithi045 ceph-mon[63880]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2024-09-17T12:01:02.493 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:02 smithi045 ceph-mon[63880]: Updating smithi165:/etc/ceph/ceph.client.admin.keyring 2024-09-17T12:01:02.493 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:02 smithi045 ceph-mon[63880]: Updating smithi045:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T12:01:02.494 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:02 smithi045 ceph-mon[63880]: Updating smithi165:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T12:01:02.494 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:02 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:02.494 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:02 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:02.494 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:02 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:02.494 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:02 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:02.494 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:02 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:02.494 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:02 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:02.716 INFO:teuthology.orchestra.run.smithi045.stdout: 5 119976 10.04 MB/sec execute 183 sec latency 38.296 ms 2024-09-17T12:01:02.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:02 smithi165 ceph-mon[25702]: Updating smithi045:/etc/ceph/ceph.conf 2024-09-17T12:01:02.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:02 smithi165 ceph-mon[25702]: Updating smithi165:/etc/ceph/ceph.conf 2024-09-17T12:01:02.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:02 smithi165 ceph-mon[25702]: Updating smithi045:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T12:01:02.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:02 smithi165 ceph-mon[25702]: Updating smithi165:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T12:01:02.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:02 smithi165 ceph-mon[25702]: Updating smithi045:/etc/ceph/ceph.client.admin.keyring 2024-09-17T12:01:02.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:02 smithi165 ceph-mon[25702]: Updating smithi165:/etc/ceph/ceph.client.admin.keyring 2024-09-17T12:01:02.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:02 smithi165 ceph-mon[25702]: Updating smithi045:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T12:01:02.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:02 smithi165 ceph-mon[25702]: Updating smithi165:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T12:01:02.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:02 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:02.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:02 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:02.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:02 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:02.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:02 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:02.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:02 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:02.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:02 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:03.513 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:03 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:01:03.513 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:03 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:01:03.513 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:03 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "quorum_status"} : dispatch 2024-09-17T12:01:03.513 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:03 smithi045 ceph-mon[63880]: pgmap v9: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 6.6 MiB/s wr, 635 op/s 2024-09-17T12:01:03.716 INFO:teuthology.orchestra.run.smithi045.stdout: 5 120366 10.04 MB/sec execute 184 sec latency 47.130 ms 2024-09-17T12:01:03.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:03 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:01:03.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:03 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:01:03.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:03 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "quorum_status"} : dispatch 2024-09-17T12:01:03.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:03 smithi165 ceph-mon[25702]: pgmap v9: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 6.6 MiB/s wr, 635 op/s 2024-09-17T12:01:04.716 INFO:teuthology.orchestra.run.smithi045.stdout: 5 120772 10.05 MB/sec execute 185 sec latency 38.464 ms 2024-09-17T12:01:05.285 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:05 smithi045 ceph-mon[63880]: Upgrade: Updating mon.smithi165 2024-09-17T12:01:05.285 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:05 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:05.285 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:05 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-17T12:01:05.285 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:05 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-17T12:01:05.285 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:05 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:05.286 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:05 smithi045 ceph-mon[63880]: Deploying daemon mon.smithi165 on smithi165 2024-09-17T12:01:05.286 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:05 smithi045 ceph-mon[63880]: pgmap v10: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 6.0 MiB/s wr, 577 op/s 2024-09-17T12:01:05.509 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:05 smithi165 ceph-mon[25702]: Upgrade: Updating mon.smithi165 2024-09-17T12:01:05.509 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:05 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:05.510 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:05 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-17T12:01:05.510 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:05 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-17T12:01:05.510 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:05 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:05.510 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:05 smithi165 ceph-mon[25702]: Deploying daemon mon.smithi165 on smithi165 2024-09-17T12:01:05.510 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:05 smithi165 ceph-mon[25702]: pgmap v10: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 22 KiB/s rd, 6.0 MiB/s wr, 577 op/s 2024-09-17T12:01:05.716 INFO:teuthology.orchestra.run.smithi045.stdout: 5 121026 10.04 MB/sec execute 186 sec latency 54.217 ms 2024-09-17T12:01:06.577 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:06 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:01:06.716 INFO:teuthology.orchestra.run.smithi045.stdout: 5 121506 10.03 MB/sec execute 187 sec latency 53.718 ms 2024-09-17T12:01:06.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:06 smithi165 ceph-mon[25702]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:01:07.211 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:06 smithi165 systemd[1]: Stopping Ceph mon.smithi165 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:01:07.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:07 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi165[25684]: 2024-09-17T12:01:07.207+0000 7f99bef35700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi165 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T12:01:07.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:07 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi165[25684]: 2024-09-17T12:01:07.207+0000 7f99bef35700 -1 mon.smithi165@1(peon) e2 *** Got Signal Terminated *** 2024-09-17T12:01:07.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:07 smithi165 podman[49224]: 2024-09-17 12:01:07.412176804 +0000 UTC m=+0.325974843 container died 9219263d3aa80a3fd08b87e3542c3e28920b346cbfd0a8a10af7f80fb389ea21 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi165, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vendor=Red Hat, Inc., GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.buildah.version=1.19.8, RELEASE=HEAD, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, architecture=x86_64, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, maintainer=Guillaume Abrioux , release=754, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-17T12:01:07.716 INFO:teuthology.orchestra.run.smithi045.stdout: 5 121871 10.03 MB/sec execute 188 sec latency 39.824 ms 2024-09-17T12:01:08.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:07 smithi165 podman[49224]: 2024-09-17 12:01:07.924970301 +0000 UTC m=+0.838768336 container cleanup 9219263d3aa80a3fd08b87e3542c3e28920b346cbfd0a8a10af7f80fb389ea21 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi165, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., vcs-type=git, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, name=centos-stream, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, io.openshift.tags=base centos centos-stream, architecture=x86_64, release=754, ceph=True) 2024-09-17T12:01:08.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:07 smithi165 bash[49224]: ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi165 2024-09-17T12:01:08.572 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:08 smithi165 podman[49235]: 2024-09-17 12:01:08.318682998 +0000 UTC m=+0.902868893 container remove 9219263d3aa80a3fd08b87e3542c3e28920b346cbfd0a8a10af7f80fb389ea21 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi165, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, ceph=True, version=8, distribution-scope=public, io.buildah.version=1.19.8) 2024-09-17T12:01:08.572 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:08 smithi165 systemd[1]: ceph-a12537de-74ea-11ef-bceb-c7b262605968@mon.smithi165.service: Deactivated successfully. 2024-09-17T12:01:08.572 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:08 smithi165 systemd[1]: Stopped Ceph mon.smithi165 for a12537de-74ea-11ef-bceb-c7b262605968. 2024-09-17T12:01:08.572 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:08 smithi165 systemd[1]: ceph-a12537de-74ea-11ef-bceb-c7b262605968@mon.smithi165.service: Consumed 7.187s CPU time. 2024-09-17T12:01:08.716 INFO:teuthology.orchestra.run.smithi045.stdout: 5 122180 10.02 MB/sec execute 189 sec latency 86.420 ms 2024-09-17T12:01:09.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:08 smithi165 systemd[1]: Starting Ceph mon.smithi165 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:01:09.366 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 podman[49353]: 2024-09-17 12:01:08.954679695 +0000 UTC m=+0.021167673 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:01:09.367 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 podman[49353]: 2024-09-17 12:01:09.061647736 +0000 UTC m=+0.128135715 container create 249eedb4123e0e62c0f100b2b23178f4290b25e31d098f9bbafd3adbd13a71e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi165, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=main-7498493, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T12:01:09.367 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 podman[49353]: 2024-09-17 12:01:09.365301606 +0000 UTC m=+0.431789582 container init 249eedb4123e0e62c0f100b2b23178f4290b25e31d098f9bbafd3adbd13a71e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi165, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=main-7498493, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-17T12:01:09.716 INFO:teuthology.orchestra.run.smithi045.stdout: 5 122531 10.02 MB/sec execute 190 sec latency 63.068 ms 2024-09-17T12:01:09.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 podman[49353]: 2024-09-17 12:01:09.371690886 +0000 UTC m=+0.438178860 container start 249eedb4123e0e62c0f100b2b23178f4290b25e31d098f9bbafd3adbd13a71e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi165, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, RELEASE=main-7498493, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-17T12:01:09.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: set uid:gid to 167:167 (ceph:ceph) 2024-09-17T12:01:09.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev), process ceph-mon, pid 2 2024-09-17T12:01:09.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: pidfile_write: ignore empty --pid-file 2024-09-17T12:01:09.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: load: jerasure load: lrc 2024-09-17T12:01:09.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: RocksDB version: 7.9.2 2024-09-17T12:01:09.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Git sha 0 2024-09-17T12:01:09.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Compile date 2024-09-15 20:52:36 2024-09-17T12:01:09.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: DB SUMMARY 2024-09-17T12:01:09.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: DB Session ID: 49LIYYWIE4R1SYSJ39X4 2024-09-17T12:01:09.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: CURRENT file: CURRENT 2024-09-17T12:01:09.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: IDENTITY file: IDENTITY 2024-09-17T12:01:09.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: MANIFEST file: MANIFEST-000009 size: 903 Bytes 2024-09-17T12:01:09.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi165/store.db dir, Total Num: 1, files: 000024.sst 2024-09-17T12:01:09.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi165/store.db: 000022.log size: 7671071 ; 2024-09-17T12:01:09.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.error_if_exists: 0 2024-09-17T12:01:09.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.create_if_missing: 0 2024-09-17T12:01:09.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.paranoid_checks: 1 2024-09-17T12:01:09.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-17T12:01:09.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-17T12:01:09.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-17T12:01:09.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.env: 0x56027bfea120 2024-09-17T12:01:09.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.fs: PosixFileSystem 2024-09-17T12:01:09.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.info_log: 0x56027e2ef8e0 2024-09-17T12:01:09.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_file_opening_threads: 16 2024-09-17T12:01:09.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.statistics: (nil) 2024-09-17T12:01:09.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.use_fsync: 0 2024-09-17T12:01:09.769 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_log_file_size: 0 2024-09-17T12:01:09.769 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T12:01:09.769 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T12:01:09.769 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.keep_log_file_num: 1000 2024-09-17T12:01:09.769 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.recycle_log_file_num: 0 2024-09-17T12:01:09.769 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.allow_fallocate: 1 2024-09-17T12:01:09.770 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.allow_mmap_reads: 0 2024-09-17T12:01:09.770 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.allow_mmap_writes: 0 2024-09-17T12:01:09.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.use_direct_reads: 0 2024-09-17T12:01:09.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T12:01:09.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.create_missing_column_families: 0 2024-09-17T12:01:09.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.db_log_dir: 2024-09-17T12:01:09.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.wal_dir: 2024-09-17T12:01:09.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T12:01:09.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T12:01:09.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T12:01:09.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T12:01:09.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T12:01:09.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T12:01:09.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.advise_random_on_open: 1 2024-09-17T12:01:09.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.db_write_buffer_size: 0 2024-09-17T12:01:09.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.write_buffer_manager: 0x56027e2f3b80 2024-09-17T12:01:09.772 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T12:01:09.772 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T12:01:09.772 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T12:01:09.772 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.rate_limiter: (nil) 2024-09-17T12:01:09.772 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T12:01:09.772 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.wal_recovery_mode: 2 2024-09-17T12:01:09.772 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.enable_thread_tracking: 0 2024-09-17T12:01:09.772 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.enable_pipelined_write: 0 2024-09-17T12:01:09.772 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.unordered_write: 0 2024-09-17T12:01:09.772 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T12:01:09.772 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T12:01:09.772 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T12:01:09.772 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T12:01:09.772 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.row_cache: None 2024-09-17T12:01:09.773 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.wal_filter: None 2024-09-17T12:01:09.773 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T12:01:09.773 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.allow_ingest_behind: 0 2024-09-17T12:01:09.773 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.two_write_queues: 0 2024-09-17T12:01:09.773 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.manual_wal_flush: 0 2024-09-17T12:01:09.773 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.wal_compression: 0 2024-09-17T12:01:09.773 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.atomic_flush: 0 2024-09-17T12:01:09.773 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T12:01:09.773 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T12:01:09.773 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T12:01:09.773 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.log_readahead_size: 0 2024-09-17T12:01:09.773 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-17T12:01:09.773 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.best_efforts_recovery: 0 2024-09-17T12:01:09.773 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-17T12:01:09.774 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-17T12:01:09.774 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.allow_data_in_errors: 0 2024-09-17T12:01:09.774 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.db_host_id: __hostname__ 2024-09-17T12:01:09.774 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-17T12:01:09.774 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_background_jobs: 2 2024-09-17T12:01:09.774 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_background_compactions: -1 2024-09-17T12:01:09.774 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_subcompactions: 1 2024-09-17T12:01:09.774 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T12:01:09.774 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T12:01:09.774 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T12:01:09.774 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_total_wal_size: 0 2024-09-17T12:01:09.775 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T12:01:09.775 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T12:01:09.775 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T12:01:09.775 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T12:01:09.775 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_open_files: -1 2024-09-17T12:01:09.775 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.bytes_per_sync: 0 2024-09-17T12:01:09.775 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T12:01:09.775 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T12:01:09.775 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.compaction_readahead_size: 0 2024-09-17T12:01:09.775 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_background_flushes: -1 2024-09-17T12:01:09.775 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Compression algorithms supported: 2024-09-17T12:01:09.775 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: kZSTD supported: 0 2024-09-17T12:01:09.775 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: kXpressCompression supported: 0 2024-09-17T12:01:09.776 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: kBZip2Compression supported: 0 2024-09-17T12:01:09.776 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T12:01:09.776 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: kLZ4Compression supported: 1 2024-09-17T12:01:09.776 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: kZlibCompression supported: 1 2024-09-17T12:01:09.776 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: kLZ4HCCompression supported: 1 2024-09-17T12:01:09.776 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: kSnappyCompression supported: 1 2024-09-17T12:01:09.776 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T12:01:09.776 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-17T12:01:09.776 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi165/store.db/MANIFEST-000009 2024-09-17T12:01:09.776 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-17T12:01:09.776 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T12:01:09.776 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.merge_operator: 2024-09-17T12:01:09.776 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.compaction_filter: None 2024-09-17T12:01:09.776 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.compaction_filter_factory: None 2024-09-17T12:01:09.777 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.sst_partitioner_factory: None 2024-09-17T12:01:09.777 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T12:01:09.777 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T12:01:09.780 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56027e2ef540) 2024-09-17T12:01:09.780 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: cache_index_and_filter_blocks: 1 2024-09-17T12:01:09.780 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T12:01:09.780 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T12:01:09.780 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: pin_top_level_index_and_filter: 1 2024-09-17T12:01:09.780 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: index_type: 0 2024-09-17T12:01:09.780 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: data_block_index_type: 0 2024-09-17T12:01:09.780 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: index_shortening: 1 2024-09-17T12:01:09.780 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-17T12:01:09.780 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: checksum: 4 2024-09-17T12:01:09.781 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: no_block_cache: 0 2024-09-17T12:01:09.781 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: block_cache: 0x56027e2e3610 2024-09-17T12:01:09.781 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: block_cache_name: BinnedLRUCache 2024-09-17T12:01:09.781 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: block_cache_options: 2024-09-17T12:01:09.781 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: capacity : 536870912 2024-09-17T12:01:09.781 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: num_shard_bits : 4 2024-09-17T12:01:09.781 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: strict_capacity_limit : 0 2024-09-17T12:01:09.781 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: high_pri_pool_ratio: 0.000 2024-09-17T12:01:09.781 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: block_cache_compressed: (nil) 2024-09-17T12:01:09.781 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: persistent_cache: (nil) 2024-09-17T12:01:09.781 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: block_size: 4096 2024-09-17T12:01:09.781 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: block_size_deviation: 10 2024-09-17T12:01:09.782 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: block_restart_interval: 16 2024-09-17T12:01:09.782 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: index_block_restart_interval: 1 2024-09-17T12:01:09.782 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: metadata_block_size: 4096 2024-09-17T12:01:09.782 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: partition_filters: 0 2024-09-17T12:01:09.782 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: use_delta_encoding: 1 2024-09-17T12:01:09.782 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: filter_policy: bloomfilter 2024-09-17T12:01:09.782 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: whole_key_filtering: 1 2024-09-17T12:01:09.782 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: verify_compression: 0 2024-09-17T12:01:09.782 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: read_amp_bytes_per_bit: 0 2024-09-17T12:01:09.782 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: format_version: 5 2024-09-17T12:01:09.782 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: enable_index_compression: 1 2024-09-17T12:01:09.782 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: block_align: 0 2024-09-17T12:01:09.782 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: max_auto_readahead_size: 262144 2024-09-17T12:01:09.783 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: prepopulate_block_cache: 0 2024-09-17T12:01:09.783 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: initial_auto_readahead_size: 8192 2024-09-17T12:01:09.783 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: num_file_reads_for_auto_readahead: 2 2024-09-17T12:01:09.783 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.write_buffer_size: 33554432 2024-09-17T12:01:09.783 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_write_buffer_number: 2 2024-09-17T12:01:09.783 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.compression: NoCompression 2024-09-17T12:01:09.783 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.bottommost_compression: Disabled 2024-09-17T12:01:09.783 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.prefix_extractor: nullptr 2024-09-17T12:01:09.783 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T12:01:09.783 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.num_levels: 7 2024-09-17T12:01:09.783 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T12:01:09.783 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T12:01:09.783 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T12:01:09.784 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T12:01:09.784 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T12:01:09.784 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T12:01:09.784 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T12:01:09.784 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T12:01:09.784 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-17T12:01:09.784 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T12:01:09.784 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-17T12:01:09.784 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-17T12:01:09.784 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T12:01:09.784 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.compression_opts.level: 32767 2024-09-17T12:01:09.784 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.compression_opts.strategy: 0 2024-09-17T12:01:09.785 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T12:01:09.785 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T12:01:09.785 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-17T12:01:09.785 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-17T12:01:09.785 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.compression_opts.enabled: false 2024-09-17T12:01:09.785 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-17T12:01:09.785 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T12:01:09.785 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T12:01:09.785 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T12:01:09.785 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.target_file_size_base: 67108864 2024-09-17T12:01:09.786 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T12:01:09.786 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T12:01:09.786 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T12:01:09.786 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T12:01:09.786 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T12:01:09.786 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T12:01:09.786 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T12:01:09.786 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T12:01:09.786 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T12:01:09.786 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T12:01:09.786 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T12:01:09.786 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T12:01:09.787 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T12:01:09.787 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-17T12:01:09.787 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.arena_block_size: 1048576 2024-09-17T12:01:09.787 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T12:01:09.787 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T12:01:09.787 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.disable_auto_compactions: 0 2024-09-17T12:01:09.787 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T12:01:09.787 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T12:01:09.787 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T12:01:09.787 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T12:01:09.787 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T12:01:09.787 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T12:01:09.787 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T12:01:09.787 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T12:01:09.788 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T12:01:09.788 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T12:01:09.788 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-17T12:01:09.788 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.inplace_update_support: 0 2024-09-17T12:01:09.788 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T12:01:09.788 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T12:01:09.788 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T12:01:09.788 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T12:01:09.788 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.bloom_locality: 0 2024-09-17T12:01:09.788 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.max_successive_merges: 0 2024-09-17T12:01:09.788 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T12:01:09.788 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.paranoid_file_checks: 0 2024-09-17T12:01:09.788 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.force_consistency_checks: 1 2024-09-17T12:01:09.789 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.report_bg_io_stats: 0 2024-09-17T12:01:09.789 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.ttl: 2592000 2024-09-17T12:01:09.789 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T12:01:09.789 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-17T12:01:09.789 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-17T12:01:09.789 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.enable_blob_files: false 2024-09-17T12:01:09.789 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.min_blob_size: 0 2024-09-17T12:01:09.789 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.blob_file_size: 268435456 2024-09-17T12:01:09.789 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-17T12:01:09.789 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-17T12:01:09.789 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-17T12:01:09.789 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-17T12:01:09.789 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-17T12:01:09.789 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.blob_file_starting_level: 0 2024-09-17T12:01:09.790 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-17T12:01:09.790 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-09-17T12:01:09.790 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T12:01:09.790 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi165/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12282, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T12:01:09.790 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-09-17T12:01:09.790 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: f9aa08a2-89b8-438c-b174-baf03273b0df 2024-09-17T12:01:09.790 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726574469405195, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-09-17T12:01:09.790 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-09-17T12:01:09.790 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726574469422267, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 4700203, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12284, "largest_seqno": 13938, "table_properties": {"data_size": 4692778, "index_size": 4652, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1733, "raw_key_size": 15582, "raw_average_key_size": 23, "raw_value_size": 4678268, "raw_average_value_size": 6982, "num_data_blocks": 211, "num_entries": 670, "num_filter_entries": 670, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726574469, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "f9aa08a2-89b8-438c-b174-baf03273b0df", "db_session_id": "49LIYYWIE4R1SYSJ39X4", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-09-17T12:01:09.790 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726574469422416, "job": 1, "event": "recovery_finished"} 2024-09-17T12:01:09.790 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-09-17T12:01:09.790 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T12:01:09.790 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi165/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-17T12:01:09.791 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x56027e312e00 2024-09-17T12:01:09.791 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: DB pointer 0x56027e414000 2024-09-17T12:01:09.791 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-17T12:01:09.791 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-17T12:01:09.791 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: ** DB Stats ** 2024-09-17T12:01:09.791 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T12:01:09.791 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T12:01:09.791 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T12:01:09.791 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T12:01:09.791 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T12:01:09.791 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T12:01:09.791 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T12:01:09.791 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: 2024-09-17T12:01:09.792 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: ** Compaction Stats [default] ** 2024-09-17T12:01:09.792 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T12:01:09.792 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-17T12:01:09.792 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: L0 1/0 4.48 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 661.8 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-17T12:01:09.792 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: L6 1/0 6.81 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-17T12:01:09.792 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Sum 2/0 11.29 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 661.8 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-17T12:01:09.792 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 661.8 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-17T12:01:09.792 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: 2024-09-17T12:01:09.792 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: ** Compaction Stats [default] ** 2024-09-17T12:01:09.792 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T12:01:09.792 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T12:01:09.792 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 661.8 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-17T12:01:09.792 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: 2024-09-17T12:01:09.793 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-17T12:01:09.793 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: 2024-09-17T12:01:09.793 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T12:01:09.793 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-09-17T12:01:09.793 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T12:01:09.793 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T12:01:09.793 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T12:01:09.793 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T12:01:09.793 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Cumulative compaction: 0.00 GB write, 197.27 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T12:01:09.793 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Interval compaction: 0.00 GB write, 197.27 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T12:01:09.793 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T12:01:09.793 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Block cache BinnedLRUCache@0x56027e2e3610#2 capacity: 512.00 MB usage: 36.05 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.4e-05 secs_since: 0 2024-09-17T12:01:09.793 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Block cache entry stats(count,size,portion): DataBlock(1,4.08 KB,0.000777841%) FilterBlock(2,10.81 KB,0.00206232%) IndexBlock(2,21.16 KB,0.00403523%) Misc(1,0.00 KB,0%) 2024-09-17T12:01:09.793 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: 2024-09-17T12:01:09.794 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T12:01:09.794 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: starting mon.smithi165 rank 1 at public addrs [v2:172.21.15.165:3300/0,v1:172.21.15.165:6789/0] at bind addrs [v2:172.21.15.165:3300/0,v1:172.21.15.165:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi165 fsid a12537de-74ea-11ef-bceb-c7b262605968 2024-09-17T12:01:09.794 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: mon.smithi165@-1(???) e2 preinit fsid a12537de-74ea-11ef-bceb-c7b262605968 2024-09-17T12:01:09.794 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: mon.smithi165@-1(???).mds e7 new map 2024-09-17T12:01:09.794 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: mon.smithi165@-1(???).mds e7 print_map 2024-09-17T12:01:09.794 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: e7 2024-09-17T12:01:09.794 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-17T12:01:09.794 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T12:01:09.794 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T12:01:09.794 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: legacy client fscid: 1 2024-09-17T12:01:09.794 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: 2024-09-17T12:01:09.794 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Filesystem 'foofs' (1) 2024-09-17T12:01:09.794 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: fs_name foofs 2024-09-17T12:01:09.794 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: epoch 5 2024-09-17T12:01:09.795 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-17T12:01:09.795 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: created 2024-09-17T11:55:22.360258+0000 2024-09-17T12:01:09.795 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: modified 2024-09-17T11:55:27.378128+0000 2024-09-17T12:01:09.795 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: tableserver 0 2024-09-17T12:01:09.795 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: root 0 2024-09-17T12:01:09.795 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: session_timeout 60 2024-09-17T12:01:09.795 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: session_autoclose 300 2024-09-17T12:01:09.795 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: max_file_size 1099511627776 2024-09-17T12:01:09.795 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: max_xattr_size 65536 2024-09-17T12:01:09.795 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: required_client_features {} 2024-09-17T12:01:09.795 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: last_failure 0 2024-09-17T12:01:09.795 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: last_failure_osd_epoch 0 2024-09-17T12:01:09.795 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T12:01:09.796 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: max_mds 1 2024-09-17T12:01:09.796 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: in 0 2024-09-17T12:01:09.796 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: up {0=14490} 2024-09-17T12:01:09.796 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: failed 2024-09-17T12:01:09.796 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: damaged 2024-09-17T12:01:09.796 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: stopped 2024-09-17T12:01:09.796 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: data_pools [3] 2024-09-17T12:01:09.796 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: metadata_pool 2 2024-09-17T12:01:09.796 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: inline_data disabled 2024-09-17T12:01:09.796 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: balancer 2024-09-17T12:01:09.796 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: bal_rank_mask -1 2024-09-17T12:01:09.796 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: standby_count_wanted 1 2024-09-17T12:01:09.796 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: qdb_cluster leader: 0 members: 2024-09-17T12:01:09.796 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: [mds.foofs.smithi045.olwaas{0:14490} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.45:6834/558522566,v1:172.21.15.45:6835/558522566] compat {c=[1],r=[1],i=[7ff]}] 2024-09-17T12:01:09.797 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: 2024-09-17T12:01:09.797 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: 2024-09-17T12:01:09.797 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: Standby daemons: 2024-09-17T12:01:09.797 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: 2024-09-17T12:01:09.797 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout: [mds.foofs.smithi165.czpmgd{-1:24315} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.165:6832/1936605703,v1:172.21.15.165:6833/1936605703] compat {c=[1],r=[1],i=[7ff]}] 2024-09-17T12:01:09.797 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: mon.smithi165@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-17T12:01:09.797 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: mon.smithi165@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T12:01:09.797 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: mon.smithi165@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T12:01:09.797 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: mon.smithi165@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T12:01:09.797 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 ceph-mon[49373]: mon.smithi165@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-09-17T12:01:09.797 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 bash[49353]: 249eedb4123e0e62c0f100b2b23178f4290b25e31d098f9bbafd3adbd13a71e7 2024-09-17T12:01:09.798 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:09 smithi165 systemd[1]: Started Ceph mon.smithi165 for a12537de-74ea-11ef-bceb-c7b262605968. 2024-09-17T12:01:10.717 INFO:teuthology.orchestra.run.smithi045.stdout: 5 122880 10.00 MB/sec execute 191 sec latency 82.582 ms 2024-09-17T12:01:11.624 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:11 smithi045 ceph-mon[63880]: mgrmap e42: smithi045.jtwyrp(active, since 19s), standbys: smithi165.ykdszc 2024-09-17T12:01:11.624 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:11 smithi045 ceph-mon[63880]: overall HEALTH_OK 2024-09-17T12:01:11.624 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:11 smithi045 ceph-mon[63880]: mon.smithi045 calling monitor election 2024-09-17T12:01:11.625 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:11 smithi045 ceph-mon[63880]: mon.smithi165 calling monitor election 2024-09-17T12:01:11.625 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:11 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mon metadata", "id": "smithi045"} : dispatch 2024-09-17T12:01:11.625 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:11 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mon metadata", "id": "smithi165"} : dispatch 2024-09-17T12:01:11.625 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:11 smithi045 ceph-mon[63880]: mon.smithi045 is new leader, mons smithi045,smithi165 in quorum (ranks 0,1) 2024-09-17T12:01:11.625 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:11 smithi045 ceph-mon[63880]: monmap epoch 3 2024-09-17T12:01:11.625 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:11 smithi045 ceph-mon[63880]: fsid a12537de-74ea-11ef-bceb-c7b262605968 2024-09-17T12:01:11.625 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:11 smithi045 ceph-mon[63880]: last_changed 2024-09-17T12:01:10.520622+0000 2024-09-17T12:01:11.625 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:11 smithi045 ceph-mon[63880]: created 2024-09-17T11:48:00.407559+0000 2024-09-17T12:01:11.625 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:11 smithi045 ceph-mon[63880]: min_mon_release 19 (squid) 2024-09-17T12:01:11.625 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:11 smithi045 ceph-mon[63880]: election_strategy: 1 2024-09-17T12:01:11.626 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:11 smithi045 ceph-mon[63880]: 0: [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] mon.smithi045 2024-09-17T12:01:11.626 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:11 smithi045 ceph-mon[63880]: 1: [v2:172.21.15.165:3300/0,v1:172.21.15.165:6789/0] mon.smithi165 2024-09-17T12:01:11.626 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:11 smithi045 ceph-mon[63880]: fsmap foofs:1 {0=foofs.smithi045.olwaas=up:active} 1 up:standby 2024-09-17T12:01:11.626 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:11 smithi045 ceph-mon[63880]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T12:01:11.626 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:11 smithi045 ceph-mon[63880]: mgrmap e42: smithi045.jtwyrp(active, since 19s), standbys: smithi165.ykdszc 2024-09-17T12:01:11.626 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:11 smithi045 ceph-mon[63880]: overall HEALTH_OK 2024-09-17T12:01:11.626 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:11 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:11.626 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:11 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:11.626 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:11 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:01:11.626 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:11 smithi045 ceph-mon[63880]: pgmap v13: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.4 MiB/s wr, 536 op/s 2024-09-17T12:01:11.717 INFO:teuthology.orchestra.run.smithi045.stdout: 5 123115 9.99 MB/sec execute 192 sec latency 72.530 ms 2024-09-17T12:01:11.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:11 smithi165 ceph-mon[49373]: mgrmap e42: smithi045.jtwyrp(active, since 19s), standbys: smithi165.ykdszc 2024-09-17T12:01:11.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:11 smithi165 ceph-mon[49373]: overall HEALTH_OK 2024-09-17T12:01:11.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:11 smithi165 ceph-mon[49373]: mon.smithi045 calling monitor election 2024-09-17T12:01:11.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:11 smithi165 ceph-mon[49373]: mon.smithi165 calling monitor election 2024-09-17T12:01:11.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:11 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mon metadata", "id": "smithi045"} : dispatch 2024-09-17T12:01:11.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:11 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mon metadata", "id": "smithi165"} : dispatch 2024-09-17T12:01:11.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:11 smithi165 ceph-mon[49373]: mon.smithi045 is new leader, mons smithi045,smithi165 in quorum (ranks 0,1) 2024-09-17T12:01:11.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:11 smithi165 ceph-mon[49373]: monmap epoch 3 2024-09-17T12:01:11.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:11 smithi165 ceph-mon[49373]: fsid a12537de-74ea-11ef-bceb-c7b262605968 2024-09-17T12:01:11.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:11 smithi165 ceph-mon[49373]: last_changed 2024-09-17T12:01:10.520622+0000 2024-09-17T12:01:11.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:11 smithi165 ceph-mon[49373]: created 2024-09-17T11:48:00.407559+0000 2024-09-17T12:01:11.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:11 smithi165 ceph-mon[49373]: min_mon_release 19 (squid) 2024-09-17T12:01:11.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:11 smithi165 ceph-mon[49373]: election_strategy: 1 2024-09-17T12:01:11.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:11 smithi165 ceph-mon[49373]: 0: [v2:172.21.15.45:3300/0,v1:172.21.15.45:6789/0] mon.smithi045 2024-09-17T12:01:11.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:11 smithi165 ceph-mon[49373]: 1: [v2:172.21.15.165:3300/0,v1:172.21.15.165:6789/0] mon.smithi165 2024-09-17T12:01:11.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:11 smithi165 ceph-mon[49373]: fsmap foofs:1 {0=foofs.smithi045.olwaas=up:active} 1 up:standby 2024-09-17T12:01:11.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:11 smithi165 ceph-mon[49373]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T12:01:11.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:11 smithi165 ceph-mon[49373]: mgrmap e42: smithi045.jtwyrp(active, since 19s), standbys: smithi165.ykdszc 2024-09-17T12:01:11.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:11 smithi165 ceph-mon[49373]: overall HEALTH_OK 2024-09-17T12:01:11.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:11 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:11.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:11 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:11.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:11 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:01:11.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:11 smithi165 ceph-mon[49373]: pgmap v13: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.4 MiB/s wr, 536 op/s 2024-09-17T12:01:12.717 INFO:teuthology.orchestra.run.smithi045.stdout: 5 123385 10.00 MB/sec execute 193 sec latency 59.042 ms 2024-09-17T12:01:13.717 INFO:teuthology.orchestra.run.smithi045.stdout: 5 123616 9.98 MB/sec execute 194 sec latency 61.461 ms 2024-09-17T12:01:13.924 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:13 smithi045 ceph-mon[63880]: pgmap v14: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 729 op/s 2024-09-17T12:01:14.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:13 smithi165 ceph-mon[49373]: pgmap v14: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.3 MiB/s wr, 729 op/s 2024-09-17T12:01:14.717 INFO:teuthology.orchestra.run.smithi045.stdout: 5 123942 9.97 MB/sec execute 195 sec latency 58.837 ms 2024-09-17T12:01:15.458 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:15 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:15.458 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:15 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:15.458 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:15 smithi045 ceph-mon[63880]: pgmap v15: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.3 MiB/s wr, 430 op/s 2024-09-17T12:01:15.458 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:15 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:15.458 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:15 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:15.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:15 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:15.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:15 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:15.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:15 smithi165 ceph-mon[49373]: pgmap v15: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.3 MiB/s wr, 430 op/s 2024-09-17T12:01:15.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:15 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:15.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:15 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:15.717 INFO:teuthology.orchestra.run.smithi045.stdout: 5 124223 9.95 MB/sec execute 196 sec latency 41.666 ms 2024-09-17T12:01:16.717 INFO:teuthology.orchestra.run.smithi045.stdout: 5 124477 9.94 MB/sec execute 197 sec latency 97.178 ms 2024-09-17T12:01:17.717 INFO:teuthology.orchestra.run.smithi045.stdout: 5 124745 9.94 MB/sec execute 198 sec latency 41.640 ms 2024-09-17T12:01:18.064 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:17 smithi045 ceph-mon[63880]: pgmap v16: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.3 MiB/s wr, 430 op/s 2024-09-17T12:01:18.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:17 smithi165 ceph-mon[49373]: pgmap v16: 97 pgs: 97 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.3 MiB/s wr, 430 op/s 2024-09-17T12:01:18.717 INFO:teuthology.orchestra.run.smithi045.stdout: 5 125263 9.94 MB/sec execute 199 sec latency 74.054 ms 2024-09-17T12:01:19.717 INFO:teuthology.orchestra.run.smithi045.stdout: 5 125661 9.93 MB/sec execute 200 sec latency 46.762 ms 2024-09-17T12:01:19.834 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:19 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:19.834 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:19 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:19.834 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:19 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:19.834 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:19 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:01:19.835 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:19 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:19.835 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:19 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:19.835 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:19 smithi045 ceph-mon[63880]: Reconfiguring mon.smithi045 (monmap changed)... 2024-09-17T12:01:19.835 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:19 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-17T12:01:19.835 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:19 smithi045 ceph-mon[63880]: pgmap v17: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.2 MiB/s wr, 601 op/s 2024-09-17T12:01:19.835 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:19 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-17T12:01:19.835 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:19 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:19.835 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:19 smithi045 ceph-mon[63880]: Reconfiguring daemon mon.smithi045 on smithi045 2024-09-17T12:01:20.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:19 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:20.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:19 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:20.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:19 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:20.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:19 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:01:20.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:19 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:20.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:19 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:20.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:19 smithi165 ceph-mon[49373]: Reconfiguring mon.smithi045 (monmap changed)... 2024-09-17T12:01:20.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:19 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-17T12:01:20.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:19 smithi165 ceph-mon[49373]: pgmap v17: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.2 MiB/s wr, 601 op/s 2024-09-17T12:01:20.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:19 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-17T12:01:20.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:19 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:20.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:19 smithi165 ceph-mon[49373]: Reconfiguring daemon mon.smithi045 on smithi045 2024-09-17T12:01:20.717 INFO:teuthology.orchestra.run.smithi045.stdout: 5 126134 9.95 MB/sec execute 201 sec latency 41.400 ms 2024-09-17T12:01:21.717 INFO:teuthology.orchestra.run.smithi045.stdout: 5 126524 9.96 MB/sec execute 202 sec latency 54.513 ms 2024-09-17T12:01:22.100 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:22 smithi045 ceph-mon[63880]: pgmap v18: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.7 MiB/s wr, 365 op/s 2024-09-17T12:01:22.100 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:22 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:22.100 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:22 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:22.100 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:22 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi045.jtwyrp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-17T12:01:22.100 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:22 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mgr services"} : dispatch 2024-09-17T12:01:22.100 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:22 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:22.101 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:22 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:01:22.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:22 smithi165 ceph-mon[49373]: pgmap v18: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.7 MiB/s wr, 365 op/s 2024-09-17T12:01:22.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:22 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:22.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:22 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:22.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:22 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi045.jtwyrp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-17T12:01:22.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:22 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mgr services"} : dispatch 2024-09-17T12:01:22.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:22 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:22.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:22 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:01:22.717 INFO:teuthology.orchestra.run.smithi045.stdout: 5 126837 9.94 MB/sec execute 203 sec latency 56.302 ms 2024-09-17T12:01:23.117 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:23 smithi045 ceph-mon[63880]: Reconfiguring mgr.smithi045.jtwyrp (monmap changed)... 2024-09-17T12:01:23.117 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:23 smithi045 ceph-mon[63880]: Reconfiguring daemon mgr.smithi045.jtwyrp on smithi045 2024-09-17T12:01:23.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:23 smithi165 ceph-mon[49373]: Reconfiguring mgr.smithi045.jtwyrp (monmap changed)... 2024-09-17T12:01:23.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:23 smithi165 ceph-mon[49373]: Reconfiguring daemon mgr.smithi045.jtwyrp on smithi045 2024-09-17T12:01:23.717 INFO:teuthology.orchestra.run.smithi045.stdout: 5 127133 9.95 MB/sec execute 204 sec latency 84.214 ms 2024-09-17T12:01:24.142 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:24 smithi045 ceph-mon[63880]: pgmap v19: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.0 MiB/s wr, 608 op/s 2024-09-17T12:01:24.142 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:24 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:24.142 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:24 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:24.142 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:24 smithi045 ceph-mon[63880]: Reconfiguring crash.smithi045 (monmap changed)... 2024-09-17T12:01:24.142 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:24 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi045", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T12:01:24.142 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:24 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:24.142 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:24 smithi045 ceph-mon[63880]: Reconfiguring daemon crash.smithi045 on smithi045 2024-09-17T12:01:24.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:24 smithi165 ceph-mon[49373]: pgmap v19: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 6.0 MiB/s wr, 608 op/s 2024-09-17T12:01:24.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:24 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:24.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:24 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:24.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:24 smithi165 ceph-mon[49373]: Reconfiguring crash.smithi045 (monmap changed)... 2024-09-17T12:01:24.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:24 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi045", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T12:01:24.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:24 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:24.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:24 smithi165 ceph-mon[49373]: Reconfiguring daemon crash.smithi045 on smithi045 2024-09-17T12:01:24.717 INFO:teuthology.orchestra.run.smithi045.stdout: 5 127472 9.94 MB/sec execute 205 sec latency 94.101 ms 2024-09-17T12:01:25.184 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:25 smithi045 ceph-mon[63880]: pgmap v20: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 414 op/s 2024-09-17T12:01:25.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:25 smithi165 ceph-mon[49373]: pgmap v20: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 414 op/s 2024-09-17T12:01:25.717 INFO:teuthology.orchestra.run.smithi045.stdout: 5 127744 9.93 MB/sec execute 206 sec latency 87.331 ms 2024-09-17T12:01:25.788 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T12:01:26.717 INFO:teuthology.orchestra.run.smithi045.stdout: 5 127955 9.92 MB/sec execute 207 sec latency 67.032 ms 2024-09-17T12:01:26.796 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T12:01:26.796 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (2m) 26s ago 12m 21.1M - 0.27.0 11f11916f8cd 2a2a54402d2a 2024-09-17T12:01:26.796 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (11m) 26s ago 11m 6413k - 17.2.0 e1d6a67b021e 8c510943b55e 2024-09-17T12:01:26.796 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (11m) 12s ago 11m 6409k - 17.2.0 e1d6a67b021e 687c24a87b70 2024-09-17T12:01:26.796 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (2m) 26s ago 11m 57.5M - dad864ee21e9 371aaf55379f 2024-09-17T12:01:26.796 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (6m) 26s ago 6m 37.4M - 17.2.0 e1d6a67b021e 6a2a4a998dd1 2024-09-17T12:01:26.796 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (5m) 12s ago 5m 15.9M - 17.2.0 e1d6a67b021e d4c2987b9bac 2024-09-17T12:01:26.796 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:8443,9283,8765 running (111s) 26s ago 13m 488M - 19.3.0-4937-g74984932 6c1fcab85602 ae2f507bd67d 2024-09-17T12:01:26.796 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283,8765 running (61s) 12s ago 11m 452M - 19.3.0-4937-g74984932 6c1fcab85602 53403866652e 2024-09-17T12:01:26.796 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (44s) 26s ago 13m 34.0M 2048M 19.3.0-4937-g74984932 6c1fcab85602 737eec78e0f0 2024-09-17T12:01:26.796 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (17s) 12s ago 11m 25.6M 2048M 19.3.0-4937-g74984932 6c1fcab85602 249eedb4123e 2024-09-17T12:01:26.797 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (5m) 26s ago 5m 257M - 4.0 e1d6a67b021e b8fc749ebb71 2024-09-17T12:01:26.797 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (5m) 12s ago 5m 83.7M - 4.0 e1d6a67b021e 771b27764432 2024-09-17T12:01:26.797 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (2m) 26s ago 11m 11.5M - 1.7.0 72c9c2088986 989b5738cb69 2024-09-17T12:01:26.797 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (2m) 12s ago 11m 11.8M - 1.7.0 72c9c2088986 e1e8ffab3f21 2024-09-17T12:01:26.797 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (10m) 26s ago 10m 457M 1937M 17.2.0 e1d6a67b021e e0ce4d29dc64 2024-09-17T12:01:26.797 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (10m) 26s ago 10m 374M 1937M 17.2.0 e1d6a67b021e da8bda7b47b3 2024-09-17T12:01:26.797 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (9m) 26s ago 9m 277M 1937M 17.2.0 e1d6a67b021e 21fe0d9e5ced 2024-09-17T12:01:26.797 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (9m) 26s ago 9m 421M 1937M 17.2.0 e1d6a67b021e 92a2389488e4 2024-09-17T12:01:26.797 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (8m) 12s ago 8m 438M 2704M 17.2.0 e1d6a67b021e e29eb74df25a 2024-09-17T12:01:26.797 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (8m) 12s ago 8m 466M 2704M 17.2.0 e1d6a67b021e 5d900eb9165b 2024-09-17T12:01:26.797 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (7m) 12s ago 7m 404M 2704M 17.2.0 e1d6a67b021e fae3e1bae466 2024-09-17T12:01:26.797 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (7m) 12s ago 7m 565M 2704M 17.2.0 e1d6a67b021e 71b9f2cdb810 2024-09-17T12:01:26.797 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (68s) 26s ago 10m 46.1M - 2.51.0 1d3b7f56885b 61b064ad6609 2024-09-17T12:01:27.007 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:26 smithi045 ceph-mon[63880]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:01:27.007 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:26 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:27.007 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:26 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:27.007 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:26 smithi045 ceph-mon[63880]: Reconfiguring osd.0 (monmap changed)... 2024-09-17T12:01:27.007 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:26 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-17T12:01:27.007 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:26 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:27.008 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:26 smithi045 ceph-mon[63880]: Reconfiguring daemon osd.0 on smithi045 2024-09-17T12:01:27.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:26 smithi165 ceph-mon[49373]: from='client.34154 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:01:27.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:26 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:27.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:26 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:27.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:26 smithi165 ceph-mon[49373]: Reconfiguring osd.0 (monmap changed)... 2024-09-17T12:01:27.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:26 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-17T12:01:27.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:26 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:27.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:26 smithi165 ceph-mon[49373]: Reconfiguring daemon osd.0 on smithi045 2024-09-17T12:01:27.357 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:01:27.358 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T12:01:27.358 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:01:27.358 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:01:27.358 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T12:01:27.358 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:01:27.358 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:01:27.358 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T12:01:27.358 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T12:01:27.359 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:01:27.359 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T12:01:27.359 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T12:01:27.359 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:01:27.359 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T12:01:27.359 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-17T12:01:27.359 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 4 2024-09-17T12:01:27.359 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T12:01:27.359 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:01:27.717 INFO:teuthology.orchestra.run.smithi045.stdout: 5 128202 9.91 MB/sec execute 208 sec latency 47.684 ms 2024-09-17T12:01:27.963 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:01:27.964 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T12:01:27.964 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T12:01:27.964 INFO:teuthology.orchestra.run.smithi045.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T12:01:27.964 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [ 2024-09-17T12:01:27.964 INFO:teuthology.orchestra.run.smithi045.stdout: "mon", 2024-09-17T12:01:27.964 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr" 2024-09-17T12:01:27.964 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-17T12:01:27.964 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "4/23 daemons upgraded", 2024-09-17T12:01:27.964 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "Currently upgrading mon daemons", 2024-09-17T12:01:27.964 INFO:teuthology.orchestra.run.smithi045.stdout: "is_paused": false 2024-09-17T12:01:27.964 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:01:28.055 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:27 smithi045 ceph-mon[63880]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:01:28.055 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:27 smithi045 ceph-mon[63880]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:01:28.055 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:27 smithi045 ceph-mon[63880]: pgmap v21: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 414 op/s 2024-09-17T12:01:28.055 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:27 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/536124463' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:01:28.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:27 smithi165 ceph-mon[49373]: from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:01:28.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:27 smithi165 ceph-mon[49373]: from='client.34162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:01:28.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:27 smithi165 ceph-mon[49373]: pgmap v21: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 414 op/s 2024-09-17T12:01:28.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:27 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/536124463' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:01:28.618 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_OK 2024-09-17T12:01:28.718 INFO:teuthology.orchestra.run.smithi045.stdout: 5 128548 9.89 MB/sec execute 209 sec latency 61.892 ms 2024-09-17T12:01:29.078 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:29 smithi045 ceph-mon[63880]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:01:29.078 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:29 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:29.078 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:29 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:29.078 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:29 smithi045 ceph-mon[63880]: Reconfiguring osd.1 (monmap changed)... 2024-09-17T12:01:29.078 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:29 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-17T12:01:29.079 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:29 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:29.079 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:29 smithi045 ceph-mon[63880]: Reconfiguring daemon osd.1 on smithi045 2024-09-17T12:01:29.079 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:29 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/3085893574' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:01:29.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:29 smithi165 ceph-mon[49373]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:01:29.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:29 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:29.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:29 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:29.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:29 smithi165 ceph-mon[49373]: Reconfiguring osd.1 (monmap changed)... 2024-09-17T12:01:29.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:29 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-17T12:01:29.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:29 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:29.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:29 smithi165 ceph-mon[49373]: Reconfiguring daemon osd.1 on smithi045 2024-09-17T12:01:29.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:29 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/3085893574' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:01:29.718 INFO:teuthology.orchestra.run.smithi045.stdout: 5 128874 9.88 MB/sec execute 210 sec latency 65.452 ms 2024-09-17T12:01:30.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:30 smithi045 ceph-mon[63880]: pgmap v22: 97 pgs: 97 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.2 MiB/s wr, 596 op/s 2024-09-17T12:01:30.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:30 smithi165 ceph-mon[49373]: pgmap v22: 97 pgs: 97 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.2 MiB/s wr, 596 op/s 2024-09-17T12:01:30.718 INFO:teuthology.orchestra.run.smithi045.stdout: 5 129095 9.86 MB/sec execute 211 sec latency 78.809 ms 2024-09-17T12:01:31.365 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:31 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:31.365 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:31 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:31.365 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:31 smithi045 ceph-mon[63880]: Reconfiguring osd.2 (monmap changed)... 2024-09-17T12:01:31.365 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:31 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-17T12:01:31.365 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:31 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:31.366 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:31 smithi045 ceph-mon[63880]: Reconfiguring daemon osd.2 on smithi045 2024-09-17T12:01:31.366 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:31 smithi045 ceph-mon[63880]: pgmap v23: 97 pgs: 97 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 424 op/s 2024-09-17T12:01:31.718 INFO:teuthology.orchestra.run.smithi045.stdout: 5 129433 9.86 MB/sec execute 212 sec latency 64.161 ms 2024-09-17T12:01:31.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:31 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:31.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:31 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:31.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:31 smithi165 ceph-mon[49373]: Reconfiguring osd.2 (monmap changed)... 2024-09-17T12:01:31.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:31 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-17T12:01:31.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:31 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:31.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:31 smithi165 ceph-mon[49373]: Reconfiguring daemon osd.2 on smithi045 2024-09-17T12:01:31.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:31 smithi165 ceph-mon[49373]: pgmap v23: 97 pgs: 97 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 424 op/s 2024-09-17T12:01:32.718 INFO:teuthology.orchestra.run.smithi045.stdout: 5 129750 9.85 MB/sec execute 213 sec latency 74.056 ms 2024-09-17T12:01:33.415 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:33 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:33.415 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:33 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:33.415 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:33 smithi045 ceph-mon[63880]: Reconfiguring osd.3 (monmap changed)... 2024-09-17T12:01:33.415 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:33 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-17T12:01:33.415 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:33 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:33.415 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:33 smithi045 ceph-mon[63880]: Reconfiguring daemon osd.3 on smithi045 2024-09-17T12:01:33.416 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:33 smithi045 ceph-mon[63880]: pgmap v24: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.0 MiB/s wr, 610 op/s 2024-09-17T12:01:33.718 INFO:teuthology.orchestra.run.smithi045.stdout: 5 130071 9.84 MB/sec execute 214 sec latency 66.988 ms 2024-09-17T12:01:33.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:33 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:33.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:33 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:33.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:33 smithi165 ceph-mon[49373]: Reconfiguring osd.3 (monmap changed)... 2024-09-17T12:01:33.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:33 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-17T12:01:33.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:33 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:33.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:33 smithi165 ceph-mon[49373]: Reconfiguring daemon osd.3 on smithi045 2024-09-17T12:01:33.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:33 smithi165 ceph-mon[49373]: pgmap v24: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.0 MiB/s wr, 610 op/s 2024-09-17T12:01:34.718 INFO:teuthology.orchestra.run.smithi045.stdout: 5 130342 9.83 MB/sec execute 215 sec latency 50.469 ms 2024-09-17T12:01:35.718 INFO:teuthology.orchestra.run.smithi045.stdout: 5 130607 9.84 MB/sec execute 216 sec latency 95.528 ms 2024-09-17T12:01:35.730 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:35 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:35.730 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:35 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:35.730 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:35 smithi045 ceph-mon[63880]: Reconfiguring mds.foofs.smithi045.olwaas (monmap changed)... 2024-09-17T12:01:35.730 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:35 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi045.olwaas", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T12:01:35.730 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:35 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:35.730 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:35 smithi045 ceph-mon[63880]: Reconfiguring daemon mds.foofs.smithi045.olwaas on smithi045 2024-09-17T12:01:35.730 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:35 smithi045 ceph-mon[63880]: pgmap v25: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.8 MiB/s wr, 367 op/s 2024-09-17T12:01:35.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:35 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:35.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:35 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:35.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:35 smithi165 ceph-mon[49373]: Reconfiguring mds.foofs.smithi045.olwaas (monmap changed)... 2024-09-17T12:01:35.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:35 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi045.olwaas", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T12:01:35.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:35 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:35.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:35 smithi165 ceph-mon[49373]: Reconfiguring daemon mds.foofs.smithi045.olwaas on smithi045 2024-09-17T12:01:35.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:35 smithi165 ceph-mon[49373]: pgmap v25: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.8 MiB/s wr, 367 op/s 2024-09-17T12:01:36.487 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:01:36.718 INFO:teuthology.orchestra.run.smithi045.stdout: 5 131049 9.85 MB/sec execute 217 sec latency 115.529 ms 2024-09-17T12:01:36.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:01:37.636 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:37 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:37.637 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:37 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:37.637 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:37 smithi045 ceph-mon[63880]: Reconfiguring crash.smithi165 (monmap changed)... 2024-09-17T12:01:37.637 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:37 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi165", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T12:01:37.637 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:37 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:37.637 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:37 smithi045 ceph-mon[63880]: Reconfiguring daemon crash.smithi165 on smithi165 2024-09-17T12:01:37.637 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:37 smithi045 ceph-mon[63880]: pgmap v26: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.8 MiB/s wr, 367 op/s 2024-09-17T12:01:37.718 INFO:teuthology.orchestra.run.smithi045.stdout: 5 131388 9.84 MB/sec execute 218 sec latency 81.274 ms 2024-09-17T12:01:38.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:37 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:38.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:37 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:38.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:37 smithi165 ceph-mon[49373]: Reconfiguring crash.smithi165 (monmap changed)... 2024-09-17T12:01:38.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:37 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi165", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T12:01:38.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:37 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:38.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:37 smithi165 ceph-mon[49373]: Reconfiguring daemon crash.smithi165 on smithi165 2024-09-17T12:01:38.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:37 smithi165 ceph-mon[49373]: pgmap v26: 97 pgs: 97 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.8 MiB/s wr, 367 op/s 2024-09-17T12:01:38.718 INFO:teuthology.orchestra.run.smithi045.stdout: 5 131751 9.85 MB/sec execute 219 sec latency 35.235 ms 2024-09-17T12:01:39.543 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:39 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:39.543 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:39 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:39.543 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:39 smithi045 ceph-mon[63880]: Reconfiguring mgr.smithi165.ykdszc (monmap changed)... 2024-09-17T12:01:39.543 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:39 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi165.ykdszc", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-17T12:01:39.544 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:39 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mgr services"} : dispatch 2024-09-17T12:01:39.544 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:39 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:39.544 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:39 smithi045 ceph-mon[63880]: Reconfiguring daemon mgr.smithi165.ykdszc on smithi165 2024-09-17T12:01:39.544 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:39 smithi045 ceph-mon[63880]: pgmap v27: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.7 MiB/s wr, 552 op/s 2024-09-17T12:01:39.718 INFO:teuthology.orchestra.run.smithi045.stdout: 5 132087 9.83 MB/sec execute 220 sec latency 55.944 ms 2024-09-17T12:01:39.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:39 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:39.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:39 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:39.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:39 smithi165 ceph-mon[49373]: Reconfiguring mgr.smithi165.ykdszc (monmap changed)... 2024-09-17T12:01:39.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:39 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi165.ykdszc", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-17T12:01:39.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:39 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mgr services"} : dispatch 2024-09-17T12:01:39.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:39 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:39.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:39 smithi165 ceph-mon[49373]: Reconfiguring daemon mgr.smithi165.ykdszc on smithi165 2024-09-17T12:01:39.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:39 smithi165 ceph-mon[49373]: pgmap v27: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.7 MiB/s wr, 552 op/s 2024-09-17T12:01:40.718 INFO:teuthology.orchestra.run.smithi045.stdout: 5 132469 9.83 MB/sec execute 221 sec latency 44.878 ms 2024-09-17T12:01:41.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:41 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:41.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:41 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:41.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:41 smithi045 ceph-mon[63880]: Reconfiguring mon.smithi165 (monmap changed)... 2024-09-17T12:01:41.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:41 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-17T12:01:41.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:41 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-17T12:01:41.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:41 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:41.312 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:41 smithi045 ceph-mon[63880]: Reconfiguring daemon mon.smithi165 on smithi165 2024-09-17T12:01:41.312 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:41 smithi045 ceph-mon[63880]: pgmap v28: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.7 MiB/s wr, 370 op/s 2024-09-17T12:01:41.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:41 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:41.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:41 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:41.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:41 smithi165 ceph-mon[49373]: Reconfiguring mon.smithi165 (monmap changed)... 2024-09-17T12:01:41.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:41 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-17T12:01:41.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:41 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-17T12:01:41.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:41 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:41.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:41 smithi165 ceph-mon[49373]: Reconfiguring daemon mon.smithi165 on smithi165 2024-09-17T12:01:41.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:41 smithi165 ceph-mon[49373]: pgmap v28: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.7 MiB/s wr, 370 op/s 2024-09-17T12:01:41.718 INFO:teuthology.orchestra.run.smithi045.stdout: 5 132913 9.84 MB/sec execute 222 sec latency 36.958 ms 2024-09-17T12:01:42.718 INFO:teuthology.orchestra.run.smithi045.stdout: 5 133230 9.82 MB/sec execute 223 sec latency 55.954 ms 2024-09-17T12:01:42.825 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:42 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:42.825 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:42 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:42.825 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:42 smithi045 ceph-mon[63880]: Reconfiguring osd.4 (monmap changed)... 2024-09-17T12:01:42.825 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:42 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-17T12:01:42.825 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:42 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:42.825 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:42 smithi045 ceph-mon[63880]: Reconfiguring daemon osd.4 on smithi165 2024-09-17T12:01:43.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:42 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:43.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:42 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:43.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:42 smithi165 ceph-mon[49373]: Reconfiguring osd.4 (monmap changed)... 2024-09-17T12:01:43.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:42 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-17T12:01:43.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:42 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:43.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:42 smithi165 ceph-mon[49373]: Reconfiguring daemon osd.4 on smithi165 2024-09-17T12:01:43.718 INFO:teuthology.orchestra.run.smithi045.stdout: 5 133563 9.83 MB/sec execute 224 sec latency 59.117 ms 2024-09-17T12:01:43.863 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:43 smithi165 ceph-mon[49373]: pgmap v29: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.0 MiB/s wr, 602 op/s 2024-09-17T12:01:43.880 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:43 smithi045 ceph-mon[63880]: pgmap v29: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.0 MiB/s wr, 602 op/s 2024-09-17T12:01:44.718 INFO:teuthology.orchestra.run.smithi045.stdout: 5 133918 9.83 MB/sec execute 225 sec latency 34.536 ms 2024-09-17T12:01:45.138 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:44 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:45.138 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:44 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:45.138 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:44 smithi045 ceph-mon[63880]: Reconfiguring osd.5 (monmap changed)... 2024-09-17T12:01:45.138 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:44 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-17T12:01:45.138 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:44 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:45.138 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:44 smithi045 ceph-mon[63880]: Reconfiguring daemon osd.5 on smithi165 2024-09-17T12:01:45.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:44 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:45.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:44 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:45.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:44 smithi165 ceph-mon[49373]: Reconfiguring osd.5 (monmap changed)... 2024-09-17T12:01:45.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:44 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-17T12:01:45.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:44 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:45.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:44 smithi165 ceph-mon[49373]: Reconfiguring daemon osd.5 on smithi165 2024-09-17T12:01:45.719 INFO:teuthology.orchestra.run.smithi045.stdout: 5 134465 9.86 MB/sec execute 226 sec latency 37.015 ms 2024-09-17T12:01:46.098 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:45 smithi165 ceph-mon[49373]: pgmap v30: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 416 op/s 2024-09-17T12:01:46.099 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:46.099 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:46.099 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-17T12:01:46.099 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:46.149 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:45 smithi045 ceph-mon[63880]: pgmap v30: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 416 op/s 2024-09-17T12:01:46.149 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:46.149 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:46.149 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-17T12:01:46.149 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:46.719 INFO:teuthology.orchestra.run.smithi045.stdout: 5 135068 9.89 MB/sec execute 227 sec latency 38.037 ms 2024-09-17T12:01:47.153 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:46 smithi045 ceph-mon[63880]: Reconfiguring osd.6 (monmap changed)... 2024-09-17T12:01:47.153 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:46 smithi045 ceph-mon[63880]: Reconfiguring daemon osd.6 on smithi165 2024-09-17T12:01:47.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:46 smithi165 ceph-mon[49373]: Reconfiguring osd.6 (monmap changed)... 2024-09-17T12:01:47.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:46 smithi165 ceph-mon[49373]: Reconfiguring daemon osd.6 on smithi165 2024-09-17T12:01:47.719 INFO:teuthology.orchestra.run.smithi045.stdout: 5 135865 9.94 MB/sec execute 228 sec latency 25.241 ms 2024-09-17T12:01:48.158 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:48 smithi045 ceph-mon[63880]: pgmap v31: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 416 op/s 2024-09-17T12:01:48.158 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:48 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:48.158 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:48 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:48.158 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:48 smithi045 ceph-mon[63880]: Reconfiguring osd.7 (monmap changed)... 2024-09-17T12:01:48.159 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:48 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-17T12:01:48.159 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:48 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:48.159 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:48 smithi045 ceph-mon[63880]: Reconfiguring daemon osd.7 on smithi165 2024-09-17T12:01:48.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:48 smithi165 ceph-mon[49373]: pgmap v31: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.3 MiB/s wr, 416 op/s 2024-09-17T12:01:48.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:48 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:48.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:48 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:48.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:48 smithi165 ceph-mon[49373]: Reconfiguring osd.7 (monmap changed)... 2024-09-17T12:01:48.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:48 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-17T12:01:48.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:48 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:48.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:48 smithi165 ceph-mon[49373]: Reconfiguring daemon osd.7 on smithi165 2024-09-17T12:01:48.719 INFO:teuthology.orchestra.run.smithi045.stdout: 5 136479 9.96 MB/sec execute 229 sec latency 36.356 ms 2024-09-17T12:01:49.668 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:49 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:49.668 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:49 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:49.668 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:49 smithi045 ceph-mon[63880]: Reconfiguring mds.foofs.smithi165.czpmgd (monmap changed)... 2024-09-17T12:01:49.668 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:49 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi165.czpmgd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T12:01:49.668 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:49 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:49.668 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:49 smithi045 ceph-mon[63880]: Reconfiguring daemon mds.foofs.smithi165.czpmgd on smithi165 2024-09-17T12:01:49.668 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:49 smithi045 ceph-mon[63880]: pgmap v32: 97 pgs: 97 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.0 MiB/s wr, 681 op/s 2024-09-17T12:01:49.719 INFO:teuthology.orchestra.run.smithi045.stdout: 5 136878 9.95 MB/sec execute 230 sec latency 32.279 ms 2024-09-17T12:01:49.953 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:49 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:49.954 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:49 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:49.954 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:49 smithi165 ceph-mon[49373]: Reconfiguring mds.foofs.smithi165.czpmgd (monmap changed)... 2024-09-17T12:01:49.954 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:49 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi165.czpmgd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T12:01:49.954 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:49 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:49.954 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:49 smithi165 ceph-mon[49373]: Reconfiguring daemon mds.foofs.smithi165.czpmgd on smithi165 2024-09-17T12:01:49.954 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:49 smithi165 ceph-mon[49373]: pgmap v32: 97 pgs: 97 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 7.0 MiB/s wr, 681 op/s 2024-09-17T12:01:50.719 INFO:teuthology.orchestra.run.smithi045.stdout: 5 137382 9.97 MB/sec execute 231 sec latency 37.359 ms 2024-09-17T12:01:51.184 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:51.184 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:51.184 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:01:51.184 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:01:51.184 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:01:51.185 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:51.185 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi045"} : dispatch 2024-09-17T12:01:51.185 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi045"}]': finished 2024-09-17T12:01:51.185 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi165"} : dispatch 2024-09-17T12:01:51.185 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi165"}]': finished 2024-09-17T12:01:51.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:51 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:51.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:51 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:51.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:51 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:01:51.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:51 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:01:51.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:51 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:01:51.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:51 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:51.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:51 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi045"} : dispatch 2024-09-17T12:01:51.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:51 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi045"}]': finished 2024-09-17T12:01:51.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:51 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi165"} : dispatch 2024-09-17T12:01:51.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:51 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi165"}]': finished 2024-09-17T12:01:51.719 INFO:teuthology.orchestra.run.smithi045.stdout: 5 137789 9.99 MB/sec execute 232 sec latency 54.142 ms 2024-09-17T12:01:52.209 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:52 smithi045 ceph-mon[63880]: Upgrade: Setting container_image for all mon 2024-09-17T12:01:52.209 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:52 smithi045 ceph-mon[63880]: pgmap v33: 97 pgs: 97 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.1 MiB/s wr, 496 op/s 2024-09-17T12:01:52.209 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:52 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:01:52.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:52 smithi165 ceph-mon[49373]: Upgrade: Setting container_image for all mon 2024-09-17T12:01:52.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:52 smithi165 ceph-mon[49373]: pgmap v33: 97 pgs: 97 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.1 MiB/s wr, 496 op/s 2024-09-17T12:01:52.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:52 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:01:52.719 INFO:teuthology.orchestra.run.smithi045.stdout: 5 138137 9.99 MB/sec execute 233 sec latency 61.369 ms 2024-09-17T12:01:53.719 INFO:teuthology.orchestra.run.smithi045.stdout: 5 138423 9.98 MB/sec execute 234 sec latency 69.061 ms 2024-09-17T12:01:54.017 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:53 smithi045 ceph-mon[63880]: Upgrade: Updating crash.smithi045 (1/2) 2024-09-17T12:01:54.018 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:53 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:54.018 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:53 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi045", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T12:01:54.018 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:53 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:54.018 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:53 smithi045 ceph-mon[63880]: Deploying daemon crash.smithi045 on smithi045 2024-09-17T12:01:54.018 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:53 smithi045 ceph-mon[63880]: pgmap v34: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 807 op/s 2024-09-17T12:01:54.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:53 smithi165 ceph-mon[49373]: Upgrade: Updating crash.smithi045 (1/2) 2024-09-17T12:01:54.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:53 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:01:54.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:53 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi045", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T12:01:54.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:53 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:01:54.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:53 smithi165 ceph-mon[49373]: Deploying daemon crash.smithi045 on smithi045 2024-09-17T12:01:54.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:53 smithi165 ceph-mon[49373]: pgmap v34: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 8.3 MiB/s wr, 807 op/s 2024-09-17T12:01:54.719 INFO:teuthology.orchestra.run.smithi045.stdout: 5 138778 9.99 MB/sec execute 235 sec latency 58.113 ms 2024-09-17T12:01:55.720 INFO:teuthology.orchestra.run.smithi045.stdout: 5 139084 9.96 MB/sec execute 236 sec latency 69.320 ms 2024-09-17T12:01:56.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:55 smithi165 ceph-mon[49373]: pgmap v35: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.9 MiB/s wr, 575 op/s 2024-09-17T12:01:56.084 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:55 smithi045 ceph-mon[63880]: pgmap v35: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.9 MiB/s wr, 575 op/s 2024-09-17T12:01:56.720 INFO:teuthology.orchestra.run.smithi045.stdout: 5 139423 9.96 MB/sec execute 237 sec latency 53.694 ms 2024-09-17T12:01:57.720 INFO:teuthology.orchestra.run.smithi045.stdout: 5 139763 9.96 MB/sec execute 238 sec latency 74.387 ms 2024-09-17T12:01:58.132 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:57 smithi045 ceph-mon[63880]: pgmap v36: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.9 MiB/s wr, 575 op/s 2024-09-17T12:01:58.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:57 smithi165 ceph-mon[49373]: pgmap v36: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.9 MiB/s wr, 575 op/s 2024-09-17T12:01:58.720 INFO:teuthology.orchestra.run.smithi045.stdout: 5 140134 9.95 MB/sec execute 239 sec latency 123.999 ms 2024-09-17T12:01:59.292 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T12:01:59.720 INFO:teuthology.orchestra.run.smithi045.stdout: 5 140483 9.94 MB/sec execute 240 sec latency 53.815 ms 2024-09-17T12:02:00.168 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:59 smithi045 ceph-mon[63880]: pgmap v37: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 779 op/s 2024-09-17T12:02:00.168 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:59 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:00.168 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:01:59 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:00.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:59 smithi165 ceph-mon[49373]: pgmap v37: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 779 op/s 2024-09-17T12:02:00.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:59 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:00.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:01:59 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:00.312 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T12:02:00.313 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (3m) 59s ago 12m 21.1M - 0.27.0 11f11916f8cd 2a2a54402d2a 2024-09-17T12:02:00.313 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 starting - - - - 2024-09-17T12:02:00.313 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (12m) 46s ago 11m 6409k - 17.2.0 e1d6a67b021e 687c24a87b70 2024-09-17T12:02:00.313 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (3m) 59s ago 12m 57.5M - dad864ee21e9 371aaf55379f 2024-09-17T12:02:00.313 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (6m) 59s ago 6m 37.4M - 17.2.0 e1d6a67b021e 6a2a4a998dd1 2024-09-17T12:02:00.313 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (6m) 46s ago 6m 15.9M - 17.2.0 e1d6a67b021e d4c2987b9bac 2024-09-17T12:02:00.313 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:8443,9283,8765 running (2m) 59s ago 13m 488M - 19.3.0-4937-g74984932 6c1fcab85602 ae2f507bd67d 2024-09-17T12:02:00.313 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283,8765 running (95s) 46s ago 11m 452M - 19.3.0-4937-g74984932 6c1fcab85602 53403866652e 2024-09-17T12:02:00.313 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (77s) 59s ago 13m 34.0M 2048M 19.3.0-4937-g74984932 6c1fcab85602 737eec78e0f0 2024-09-17T12:02:00.313 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (51s) 46s ago 11m 25.6M 2048M 19.3.0-4937-g74984932 6c1fcab85602 249eedb4123e 2024-09-17T12:02:00.313 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (6m) 59s ago 6m 257M - 4.0 e1d6a67b021e b8fc749ebb71 2024-09-17T12:02:00.314 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (6m) 46s ago 6m 83.7M - 4.0 e1d6a67b021e 771b27764432 2024-09-17T12:02:00.314 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (3m) 59s ago 12m 11.5M - 1.7.0 72c9c2088986 989b5738cb69 2024-09-17T12:02:00.314 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (2m) 46s ago 11m 11.8M - 1.7.0 72c9c2088986 e1e8ffab3f21 2024-09-17T12:02:00.314 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (11m) 59s ago 11m 457M 1937M 17.2.0 e1d6a67b021e e0ce4d29dc64 2024-09-17T12:02:00.314 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (10m) 59s ago 10m 374M 1937M 17.2.0 e1d6a67b021e da8bda7b47b3 2024-09-17T12:02:00.314 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (10m) 59s ago 10m 277M 1937M 17.2.0 e1d6a67b021e 21fe0d9e5ced 2024-09-17T12:02:00.314 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (9m) 59s ago 9m 421M 1937M 17.2.0 e1d6a67b021e 92a2389488e4 2024-09-17T12:02:00.314 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (9m) 46s ago 9m 438M 2704M 17.2.0 e1d6a67b021e e29eb74df25a 2024-09-17T12:02:00.314 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (8m) 46s ago 8m 466M 2704M 17.2.0 e1d6a67b021e 5d900eb9165b 2024-09-17T12:02:00.314 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (8m) 46s ago 8m 404M 2704M 17.2.0 e1d6a67b021e fae3e1bae466 2024-09-17T12:02:00.314 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (8m) 46s ago 8m 565M 2704M 17.2.0 e1d6a67b021e 71b9f2cdb810 2024-09-17T12:02:00.314 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (102s) 59s ago 11m 46.1M - 2.51.0 1d3b7f56885b 61b064ad6609 2024-09-17T12:02:00.720 INFO:teuthology.orchestra.run.smithi045.stdout: 5 140849 9.95 MB/sec execute 241 sec latency 65.209 ms 2024-09-17T12:02:00.934 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:00 smithi045 ceph-mon[63880]: from='client.34178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:02:00.934 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:00 smithi045 ceph-mon[63880]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:02:00.934 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:00 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:00.934 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:00 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi165", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T12:02:00.934 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:00 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:02:00.970 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:02:00.970 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T12:02:00.971 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:02:00.971 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:02:00.971 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T12:02:00.971 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:02:00.971 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:02:00.971 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T12:02:00.971 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T12:02:00.971 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:02:00.971 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T12:02:00.971 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T12:02:00.971 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:02:00.971 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T12:02:00.971 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-17T12:02:00.971 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 4 2024-09-17T12:02:00.972 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T12:02:00.972 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:02:01.234 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:00 smithi165 ceph-mon[49373]: from='client.34178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:02:01.234 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:00 smithi165 ceph-mon[49373]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:02:01.234 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:00 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:01.234 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:00 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi165", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-17T12:02:01.234 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:00 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:02:01.444 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:02:01.444 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T12:02:01.444 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T12:02:01.444 INFO:teuthology.orchestra.run.smithi045.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T12:02:01.444 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [ 2024-09-17T12:02:01.444 INFO:teuthology.orchestra.run.smithi045.stdout: "mon", 2024-09-17T12:02:01.444 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr" 2024-09-17T12:02:01.444 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-17T12:02:01.445 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "4/23 daemons upgraded", 2024-09-17T12:02:01.445 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "Currently upgrading crash daemons", 2024-09-17T12:02:01.445 INFO:teuthology.orchestra.run.smithi045.stdout: "is_paused": false 2024-09-17T12:02:01.445 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:02:01.720 INFO:teuthology.orchestra.run.smithi045.stdout: 5 141192 9.95 MB/sec execute 242 sec latency 93.245 ms 2024-09-17T12:02:02.011 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:01 smithi045 ceph-mon[63880]: from='client.44125 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:02:02.011 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:01 smithi045 ceph-mon[63880]: pgmap v38: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.3 MiB/s wr, 514 op/s 2024-09-17T12:02:02.011 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:01 smithi045 ceph-mon[63880]: Upgrade: Updating crash.smithi165 (2/2) 2024-09-17T12:02:02.011 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:01 smithi045 ceph-mon[63880]: Deploying daemon crash.smithi165 on smithi165 2024-09-17T12:02:02.011 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:01 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/849261274' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:02:02.015 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_OK 2024-09-17T12:02:02.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:01 smithi165 ceph-mon[49373]: from='client.44125 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:02:02.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:01 smithi165 ceph-mon[49373]: pgmap v38: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.3 MiB/s wr, 514 op/s 2024-09-17T12:02:02.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:01 smithi165 ceph-mon[49373]: Upgrade: Updating crash.smithi165 (2/2) 2024-09-17T12:02:02.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:01 smithi165 ceph-mon[49373]: Deploying daemon crash.smithi165 on smithi165 2024-09-17T12:02:02.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:01 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/849261274' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:02:02.720 INFO:teuthology.orchestra.run.smithi045.stdout: 5 141433 9.94 MB/sec execute 243 sec latency 88.237 ms 2024-09-17T12:02:03.066 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:02 smithi045 ceph-mon[63880]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:02:03.066 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:02 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/936012506' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:02:03.191 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:02 smithi165 ceph-mon[49373]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:02:03.192 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:02 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/936012506' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:02:03.720 INFO:teuthology.orchestra.run.smithi045.stdout: 5 141669 9.93 MB/sec execute 244 sec latency 96.267 ms 2024-09-17T12:02:04.103 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:03 smithi045 ceph-mon[63880]: pgmap v39: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 724 op/s 2024-09-17T12:02:04.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:03 smithi165 ceph-mon[49373]: pgmap v39: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 724 op/s 2024-09-17T12:02:04.720 INFO:teuthology.orchestra.run.smithi045.stdout: 5 141919 9.91 MB/sec execute 245 sec latency 66.283 ms 2024-09-17T12:02:05.636 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:05 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:05.636 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:05 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:05.636 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:05 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:02:05.636 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:05 smithi045 ceph-mon[63880]: pgmap v40: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 414 op/s 2024-09-17T12:02:05.720 INFO:teuthology.orchestra.run.smithi045.stdout: 5 142135 9.90 MB/sec execute 246 sec latency 93.131 ms 2024-09-17T12:02:05.869 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:05 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:05.870 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:05 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:05.870 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:05 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:02:05.870 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:05 smithi165 ceph-mon[49373]: pgmap v40: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 414 op/s 2024-09-17T12:02:06.701 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:06 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:02:06.720 INFO:teuthology.orchestra.run.smithi045.stdout: 5 142371 9.89 MB/sec execute 247 sec latency 79.571 ms 2024-09-17T12:02:06.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:06 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:02:07.721 INFO:teuthology.orchestra.run.smithi045.stdout: 5 142633 9.87 MB/sec execute 248 sec latency 45.905 ms 2024-09-17T12:02:07.732 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:07 smithi045 ceph-mon[63880]: pgmap v41: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 414 op/s 2024-09-17T12:02:08.001 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:07 smithi165 ceph-mon[49373]: pgmap v41: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.3 MiB/s wr, 414 op/s 2024-09-17T12:02:08.721 INFO:teuthology.orchestra.run.smithi045.stdout: 5 142860 9.84 MB/sec execute 249 sec latency 50.575 ms 2024-09-17T12:02:09.556 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:09 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:09.556 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:09 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:09.556 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:09 smithi045 ceph-mon[63880]: pgmap v42: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.8 MiB/s wr, 559 op/s 2024-09-17T12:02:09.556 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:09 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:09.557 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:09 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:09.721 INFO:teuthology.orchestra.run.smithi045.stdout: 5 143248 9.85 MB/sec execute 250 sec latency 44.067 ms 2024-09-17T12:02:09.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:09 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:09.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:09 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:09.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:09 smithi165 ceph-mon[49373]: pgmap v42: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.8 MiB/s wr, 559 op/s 2024-09-17T12:02:09.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:09 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:09.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:09 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:10.721 INFO:teuthology.orchestra.run.smithi045.stdout: 5 143612 9.85 MB/sec execute 251 sec latency 40.308 ms 2024-09-17T12:02:11.722 INFO:teuthology.orchestra.run.smithi045.stdout: 5 143982 9.84 MB/sec execute 252 sec latency 75.509 ms 2024-09-17T12:02:12.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:11 smithi045 ceph-mon[63880]: pgmap v43: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.6 MiB/s wr, 355 op/s 2024-09-17T12:02:12.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:11 smithi165 ceph-mon[49373]: pgmap v43: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.6 MiB/s wr, 355 op/s 2024-09-17T12:02:12.723 INFO:teuthology.orchestra.run.smithi045.stdout: 5 144372 9.85 MB/sec execute 253 sec latency 29.957 ms 2024-09-17T12:02:13.375 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:13 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:13.375 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:13 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:13.375 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:13 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:13.375 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:13 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:13.375 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:13 smithi045 ceph-mon[63880]: pgmap v44: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.5 MiB/s wr, 555 op/s 2024-09-17T12:02:13.375 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:13 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:13.376 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:13 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:13.722 INFO:teuthology.orchestra.run.smithi045.stdout: 5 144742 9.86 MB/sec execute 254 sec latency 50.558 ms 2024-09-17T12:02:13.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:13 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:13.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:13 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:13.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:13 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:13.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:13 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:13.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:13 smithi165 ceph-mon[49373]: pgmap v44: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.5 MiB/s wr, 555 op/s 2024-09-17T12:02:13.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:13 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:13.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:13 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:14.722 INFO:teuthology.orchestra.run.smithi045.stdout: 5 145119 9.86 MB/sec execute 255 sec latency 53.310 ms 2024-09-17T12:02:15.722 INFO:teuthology.orchestra.run.smithi045.stdout: 5 145444 9.86 MB/sec execute 256 sec latency 35.020 ms 2024-09-17T12:02:15.903 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:15 smithi045 ceph-mon[63880]: pgmap v45: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.4 MiB/s wr, 344 op/s 2024-09-17T12:02:16.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:15 smithi165 ceph-mon[49373]: pgmap v45: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.4 MiB/s wr, 344 op/s 2024-09-17T12:02:16.723 INFO:teuthology.orchestra.run.smithi045.stdout: 5 145628 9.84 MB/sec execute 257 sec latency 50.933 ms 2024-09-17T12:02:17.723 INFO:teuthology.orchestra.run.smithi045.stdout: 5 145844 9.83 MB/sec execute 258 sec latency 74.672 ms 2024-09-17T12:02:17.990 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:17 smithi045 ceph-mon[63880]: pgmap v46: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.4 MiB/s wr, 344 op/s 2024-09-17T12:02:18.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:17 smithi165 ceph-mon[49373]: pgmap v46: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.4 MiB/s wr, 344 op/s 2024-09-17T12:02:18.725 INFO:teuthology.orchestra.run.smithi045.stdout: 5 146107 9.81 MB/sec execute 259 sec latency 79.758 ms 2024-09-17T12:02:19.723 INFO:teuthology.orchestra.run.smithi045.stdout: 5 146349 9.79 MB/sec execute 260 sec latency 69.677 ms 2024-09-17T12:02:20.027 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:19 smithi045 ceph-mon[63880]: pgmap v47: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.6 MiB/s wr, 542 op/s 2024-09-17T12:02:20.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:19 smithi165 ceph-mon[49373]: pgmap v47: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.6 MiB/s wr, 542 op/s 2024-09-17T12:02:20.723 INFO:teuthology.orchestra.run.smithi045.stdout: 5 146568 9.79 MB/sec execute 261 sec latency 91.409 ms 2024-09-17T12:02:21.295 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:21 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:21.296 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:21 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:21.296 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:21 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:02:21.296 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:21 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:02:21.297 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:21 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:21.297 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:21 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:21.297 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:21 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:02:21.297 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:21 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:02:21.297 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:21 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:02:21.297 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:21 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:02:21.297 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:21 smithi045 ceph-mon[63880]: Upgrade: Setting container_image for all crash 2024-09-17T12:02:21.297 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:21 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:21.298 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:21 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi045"} : dispatch 2024-09-17T12:02:21.298 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:21 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi045"}]': finished 2024-09-17T12:02:21.298 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:21 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi165"} : dispatch 2024-09-17T12:02:21.298 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:21 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi165"}]': finished 2024-09-17T12:02:21.298 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:21 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-17T12:02:21.298 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:21 smithi045 ceph-mon[63880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T12:02:21.298 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:21 smithi045 ceph-mon[63880]: Upgrade: osd.0 is safe to restart 2024-09-17T12:02:21.298 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:21 smithi045 ceph-mon[63880]: pgmap v48: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 396 op/s 2024-09-17T12:02:21.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:21 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:21.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:21 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:21.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:21 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:02:21.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:21 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:02:21.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:21 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:21.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:21 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:21.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:21 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:02:21.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:21 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:02:21.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:21 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:02:21.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:21 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:02:21.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:21 smithi165 ceph-mon[49373]: Upgrade: Setting container_image for all crash 2024-09-17T12:02:21.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:21 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:21.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:21 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi045"} : dispatch 2024-09-17T12:02:21.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:21 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi045"}]': finished 2024-09-17T12:02:21.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:21 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi165"} : dispatch 2024-09-17T12:02:21.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:21 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi165"}]': finished 2024-09-17T12:02:21.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:21 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-17T12:02:21.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:21 smithi165 ceph-mon[49373]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T12:02:21.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:21 smithi165 ceph-mon[49373]: Upgrade: osd.0 is safe to restart 2024-09-17T12:02:21.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:21 smithi165 ceph-mon[49373]: pgmap v48: 97 pgs: 97 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 396 op/s 2024-09-17T12:02:21.723 INFO:teuthology.orchestra.run.smithi045.stdout: 5 146776 9.76 MB/sec execute 262 sec latency 68.192 ms 2024-09-17T12:02:22.337 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:22 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:02:22.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:22 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:02:22.723 INFO:teuthology.orchestra.run.smithi045.stdout: 5 147053 9.76 MB/sec execute 263 sec latency 83.775 ms 2024-09-17T12:02:23.734 INFO:teuthology.orchestra.run.smithi045.stdout: 5 147349 9.74 MB/sec execute 264 sec latency 43.085 ms 2024-09-17T12:02:24.131 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:23 smithi045 ceph-mon[63880]: pgmap v49: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.4 MiB/s wr, 546 op/s 2024-09-17T12:02:24.131 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:23 smithi045 ceph-mon[63880]: Upgrade: Updating osd.0 2024-09-17T12:02:24.131 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:23 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:24.131 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:23 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-17T12:02:24.131 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:23 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:02:24.132 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:23 smithi045 ceph-mon[63880]: Deploying daemon osd.0 on smithi045 2024-09-17T12:02:24.132 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:23 smithi045 ceph-mon[63880]: mgrmap e43: smithi045.jtwyrp(active, since 92s), standbys: smithi165.ykdszc 2024-09-17T12:02:24.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:23 smithi165 ceph-mon[49373]: pgmap v49: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.4 MiB/s wr, 546 op/s 2024-09-17T12:02:24.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:23 smithi165 ceph-mon[49373]: Upgrade: Updating osd.0 2024-09-17T12:02:24.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:23 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:24.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:23 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-17T12:02:24.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:23 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:02:24.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:23 smithi165 ceph-mon[49373]: Deploying daemon osd.0 on smithi045 2024-09-17T12:02:24.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:23 smithi165 ceph-mon[49373]: mgrmap e43: smithi045.jtwyrp(active, since 92s), standbys: smithi165.ykdszc 2024-09-17T12:02:24.741 INFO:teuthology.orchestra.run.smithi045.stdout: 5 147678 9.74 MB/sec execute 265 sec latency 53.857 ms 2024-09-17T12:02:25.740 INFO:teuthology.orchestra.run.smithi045.stdout: 5 147974 9.73 MB/sec execute 266 sec latency 64.985 ms 2024-09-17T12:02:25.932 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:25 smithi045 ceph-mon[63880]: pgmap v50: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.5 MiB/s wr, 346 op/s 2024-09-17T12:02:26.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:25 smithi165 ceph-mon[49373]: pgmap v50: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.5 MiB/s wr, 346 op/s 2024-09-17T12:02:26.740 INFO:teuthology.orchestra.run.smithi045.stdout: 5 148297 9.74 MB/sec execute 267 sec latency 87.265 ms 2024-09-17T12:02:27.509 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:27 smithi045 systemd[1]: Stopping Ceph osd.0 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:02:27.740 INFO:teuthology.orchestra.run.smithi045.stdout: 5 148784 9.76 MB/sec execute 268 sec latency 53.347 ms 2024-09-17T12:02:28.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:27 smithi045 ceph-mon[63880]: pgmap v51: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.5 MiB/s wr, 346 op/s 2024-09-17T12:02:28.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:27 smithi045 ceph-mon[63880]: osd.0 marked itself down and dead 2024-09-17T12:02:28.090 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:27 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0[34164]: 2024-09-17T12:02:27.853+0000 7faab6288700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T12:02:28.090 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:27 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0[34164]: 2024-09-17T12:02:27.853+0000 7faab6288700 -1 osd.0 72 *** Got signal Terminated *** 2024-09-17T12:02:28.090 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:27 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0[34164]: 2024-09-17T12:02:27.853+0000 7faab6288700 -1 osd.0 72 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T12:02:28.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:27 smithi165 ceph-mon[49373]: pgmap v51: 97 pgs: 97 active+clean; 341 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.5 MiB/s wr, 346 op/s 2024-09-17T12:02:28.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:27 smithi165 ceph-mon[49373]: osd.0 marked itself down and dead 2024-09-17T12:02:28.740 INFO:teuthology.orchestra.run.smithi045.stdout: 5 148873 9.73 MB/sec execute 269 sec latency 778.496 ms 2024-09-17T12:02:28.839 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:28 smithi045 podman[68224]: 2024-09-17 12:02:28.659445674 +0000 UTC m=+0.972292030 container died e0ce4d29dc6463fccd092d6caa7c40cfd22b9d9df61bf4caee01c2b36d6a6564 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , release=754, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, version=8) 2024-09-17T12:02:29.185 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:28 smithi045 ceph-mon[63880]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T12:02:29.185 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:28 smithi045 ceph-mon[63880]: osdmap e73: 8 total, 7 up, 8 in 2024-09-17T12:02:29.185 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:29 smithi045 podman[68224]: 2024-09-17 12:02:29.054532439 +0000 UTC m=+1.367378784 container cleanup e0ce4d29dc6463fccd092d6caa7c40cfd22b9d9df61bf4caee01c2b36d6a6564 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, release=754, GIT_BRANCH=HEAD, ceph=True, distribution-scope=public, architecture=x86_64, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.component=centos-stream-container) 2024-09-17T12:02:29.186 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:29 smithi045 bash[68224]: ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0 2024-09-17T12:02:29.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:28 smithi165 ceph-mon[49373]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T12:02:29.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:28 smithi165 ceph-mon[49373]: osdmap e73: 8 total, 7 up, 8 in 2024-09-17T12:02:29.436 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:29 smithi045 podman[68239]: 2024-09-17 12:02:29.319521017 +0000 UTC m=+0.659362175 container remove e0ce4d29dc6463fccd092d6caa7c40cfd22b9d9df61bf4caee01c2b36d6a6564 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, ceph=True, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, vendor=Red Hat, Inc., name=centos-stream, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, RELEASE=HEAD, release=754, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0) 2024-09-17T12:02:29.740 INFO:teuthology.orchestra.run.smithi045.stdout: 5 149236 9.74 MB/sec execute 270 sec latency 1043.055 ms 2024-09-17T12:02:30.197 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:29 smithi045 ceph-mon[63880]: pgmap v53: 97 pgs: 13 stale+active+clean, 84 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 401 op/s 2024-09-17T12:02:30.197 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:29 smithi045 ceph-mon[63880]: osdmap e74: 8 total, 7 up, 8 in 2024-09-17T12:02:30.197 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:29 smithi045 podman[68318]: 2024-09-17 12:02:29.875900834 +0000 UTC m=+0.025733478 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:02:30.198 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:30 smithi045 podman[68318]: 2024-09-17 12:02:30.017878188 +0000 UTC m=+0.167710820 container create 0b39242fe0fdbdcfaad422ee3e0d0517442f39b3cb09093a06615aea78e343f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-17T12:02:30.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:29 smithi165 ceph-mon[49373]: pgmap v53: 97 pgs: 13 stale+active+clean, 84 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 401 op/s 2024-09-17T12:02:30.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:29 smithi165 ceph-mon[49373]: osdmap e74: 8 total, 7 up, 8 in 2024-09-17T12:02:30.448 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:30 smithi045 podman[68318]: 2024-09-17 12:02:30.411657559 +0000 UTC m=+0.561490186 container init 0b39242fe0fdbdcfaad422ee3e0d0517442f39b3cb09093a06615aea78e343f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=main-7498493, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-17T12:02:30.448 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:30 smithi045 podman[68318]: 2024-09-17 12:02:30.41981737 +0000 UTC m=+0.569649997 container start 0b39242fe0fdbdcfaad422ee3e0d0517442f39b3cb09093a06615aea78e343f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=main-7498493, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T12:02:30.707 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:30 smithi045 podman[68318]: 2024-09-17 12:02:30.549430244 +0000 UTC m=+0.699262870 container attach 0b39242fe0fdbdcfaad422ee3e0d0517442f39b3cb09093a06615aea78e343f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-17T12:02:30.740 INFO:teuthology.orchestra.run.smithi045.stdout: 5 149669 9.73 MB/sec execute 271 sec latency 42.684 ms 2024-09-17T12:02:30.958 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:30 smithi045 podman[68318]: 2024-09-17 12:02:30.763800721 +0000 UTC m=+0.913633365 container died 0b39242fe0fdbdcfaad422ee3e0d0517442f39b3cb09093a06615aea78e343f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493) 2024-09-17T12:02:31.743 INFO:teuthology.orchestra.run.smithi045.stdout: 5 149920 9.71 MB/sec execute 272 sec latency 64.666 ms 2024-09-17T12:02:31.747 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:31 smithi045 podman[68340]: 2024-09-17 12:02:31.592004673 +0000 UTC m=+0.819635426 container remove 0b39242fe0fdbdcfaad422ee3e0d0517442f39b3cb09093a06615aea78e343f1 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=main-7498493, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T12:02:31.747 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:31 smithi045 systemd[1]: ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-17T12:02:31.747 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:31 smithi045 systemd[1]: Stopped Ceph osd.0 for a12537de-74ea-11ef-bceb-c7b262605968. 2024-09-17T12:02:31.747 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:31 smithi045 systemd[1]: ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.0.service: Consumed 2min 32.150s CPU time. 2024-09-17T12:02:32.004 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:31 smithi045 ceph-mon[63880]: pgmap v55: 97 pgs: 13 stale+active+clean, 84 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.9 MiB/s wr, 278 op/s 2024-09-17T12:02:32.256 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:32 smithi045 systemd[1]: Starting Ceph osd.0 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:02:32.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:31 smithi165 ceph-mon[49373]: pgmap v55: 97 pgs: 13 stale+active+clean, 84 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 2.9 MiB/s wr, 278 op/s 2024-09-17T12:02:32.512 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:32 smithi045 podman[68440]: 2024-09-17 12:02:32.212832245 +0000 UTC m=+0.027296703 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:02:32.512 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:32 smithi045 podman[68440]: 2024-09-17 12:02:32.454645473 +0000 UTC m=+0.269109911 container create 474097ded63b91d38edb5a40f98a68deeb6020ca970aeea792bb8fff129660e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=main-7498493, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T12:02:32.595 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T12:02:32.743 INFO:teuthology.orchestra.run.smithi045.stdout: 5 150254 9.71 MB/sec execute 273 sec latency 76.903 ms 2024-09-17T12:02:33.026 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:32 smithi045 podman[68440]: 2024-09-17 12:02:32.948801958 +0000 UTC m=+0.763266399 container init 474097ded63b91d38edb5a40f98a68deeb6020ca970aeea792bb8fff129660e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T12:02:33.027 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:32 smithi045 podman[68440]: 2024-09-17 12:02:32.963697294 +0000 UTC m=+0.778161736 container start 474097ded63b91d38edb5a40f98a68deeb6020ca970aeea792bb8fff129660e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-17T12:02:33.291 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:33 smithi045 ceph-mon[63880]: Health check failed: Degraded data redundancy: 228/1617 objects degraded (14.100%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T12:02:33.291 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:33 smithi045 podman[68440]: 2024-09-17 12:02:33.099034482 +0000 UTC m=+0.913498937 container attach 474097ded63b91d38edb5a40f98a68deeb6020ca970aeea792bb8fff129660e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=main-7498493, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-17T12:02:33.291 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:33 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-activate[68476]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:02:33.291 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:33 smithi045 bash[68440]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:02:33.291 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:33 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-activate[68476]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:02:33.292 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:33 smithi045 bash[68440]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:02:33.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:33 smithi165 ceph-mon[49373]: Health check failed: Degraded data redundancy: 228/1617 objects degraded (14.100%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T12:02:33.743 INFO:teuthology.orchestra.run.smithi045.stdout: 5 150646 9.72 MB/sec execute 274 sec latency 38.050 ms 2024-09-17T12:02:33.779 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T12:02:33.779 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (3m) 21s ago 13m 20.4M - 0.27.0 11f11916f8cd 2a2a54402d2a 2024-09-17T12:02:33.779 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (35s) 21s ago 13m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 e545cbf7736d 2024-09-17T12:02:33.779 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (29s) 25s ago 12m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 c1e89e114e4d 2024-09-17T12:02:33.780 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (3m) 21s ago 12m 57.4M - dad864ee21e9 371aaf55379f 2024-09-17T12:02:33.780 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (7m) 21s ago 7m 37.3M - 17.2.0 e1d6a67b021e 6a2a4a998dd1 2024-09-17T12:02:33.780 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (7m) 25s ago 7m 16.0M - 17.2.0 e1d6a67b021e d4c2987b9bac 2024-09-17T12:02:33.780 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:8443,9283,8765 running (2m) 21s ago 14m 509M - 19.3.0-4937-g74984932 6c1fcab85602 ae2f507bd67d 2024-09-17T12:02:33.780 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283,8765 running (2m) 25s ago 12m 453M - 19.3.0-4937-g74984932 6c1fcab85602 53403866652e 2024-09-17T12:02:33.780 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (111s) 21s ago 14m 44.4M 2048M 19.3.0-4937-g74984932 6c1fcab85602 737eec78e0f0 2024-09-17T12:02:33.780 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (84s) 25s ago 12m 36.1M 2048M 19.3.0-4937-g74984932 6c1fcab85602 249eedb4123e 2024-09-17T12:02:33.780 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (6m) 21s ago 6m 260M - 4.0 e1d6a67b021e b8fc749ebb71 2024-09-17T12:02:33.780 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (6m) 25s ago 6m 83.8M - 4.0 e1d6a67b021e 771b27764432 2024-09-17T12:02:33.780 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (3m) 21s ago 12m 11.1M - 1.7.0 72c9c2088986 989b5738cb69 2024-09-17T12:02:33.780 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (3m) 25s ago 12m 11.9M - 1.7.0 72c9c2088986 e1e8ffab3f21 2024-09-17T12:02:33.781 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (11m) 21s ago 11m 516M 1937M 17.2.0 e1d6a67b021e e0ce4d29dc64 2024-09-17T12:02:33.781 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (11m) 21s ago 11m 449M 1937M 17.2.0 e1d6a67b021e da8bda7b47b3 2024-09-17T12:02:33.781 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (10m) 21s ago 10m 322M 1937M 17.2.0 e1d6a67b021e 21fe0d9e5ced 2024-09-17T12:02:33.781 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (10m) 21s ago 10m 479M 1937M 17.2.0 e1d6a67b021e 92a2389488e4 2024-09-17T12:02:33.781 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (9m) 25s ago 9m 494M 2704M 17.2.0 e1d6a67b021e e29eb74df25a 2024-09-17T12:02:33.781 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (9m) 25s ago 9m 544M 2704M 17.2.0 e1d6a67b021e 5d900eb9165b 2024-09-17T12:02:33.781 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (9m) 25s ago 9m 466M 2704M 17.2.0 e1d6a67b021e fae3e1bae466 2024-09-17T12:02:33.781 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (8m) 25s ago 8m 371M 2704M 17.2.0 e1d6a67b021e 71b9f2cdb810 2024-09-17T12:02:33.781 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (2m) 21s ago 12m 53.6M - 2.51.0 1d3b7f56885b 61b064ad6609 2024-09-17T12:02:34.052 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:34 smithi045 ceph-mon[63880]: from='client.44139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:02:34.052 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:34 smithi045 ceph-mon[63880]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.9 MiB/s wr, 564 op/s; 228/1617 objects degraded (14.100%) 2024-09-17T12:02:34.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:34 smithi165 ceph-mon[49373]: from='client.44139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:02:34.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:34 smithi165 ceph-mon[49373]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.9 MiB/s wr, 564 op/s; 228/1617 objects degraded (14.100%) 2024-09-17T12:02:34.526 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:02:34.526 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T12:02:34.526 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:02:34.526 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:02:34.526 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T12:02:34.526 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:02:34.526 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:02:34.526 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T12:02:34.526 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7 2024-09-17T12:02:34.526 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:02:34.527 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T12:02:34.527 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T12:02:34.527 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:02:34.527 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T12:02:34.527 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-17T12:02:34.527 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 4 2024-09-17T12:02:34.527 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T12:02:34.527 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:02:34.743 INFO:teuthology.orchestra.run.smithi045.stdout: 5 151186 9.73 MB/sec execute 275 sec latency 67.833 ms 2024-09-17T12:02:34.805 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:34 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-activate[68476]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T12:02:34.805 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:34 smithi045 bash[68440]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T12:02:34.805 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:34 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-activate[68476]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T12:02:34.805 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:34 smithi045 bash[68440]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T12:02:35.044 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:02:35.044 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T12:02:35.044 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T12:02:35.044 INFO:teuthology.orchestra.run.smithi045.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T12:02:35.044 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [ 2024-09-17T12:02:35.045 INFO:teuthology.orchestra.run.smithi045.stdout: "crash", 2024-09-17T12:02:35.045 INFO:teuthology.orchestra.run.smithi045.stdout: "mon", 2024-09-17T12:02:35.045 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr" 2024-09-17T12:02:35.045 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-17T12:02:35.045 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "6/23 daemons upgraded", 2024-09-17T12:02:35.045 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T12:02:35.045 INFO:teuthology.orchestra.run.smithi045.stdout: "is_paused": false 2024-09-17T12:02:35.045 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:02:35.057 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:35 smithi045 ceph-mon[63880]: from='client.44143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:02:35.057 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:35 smithi045 ceph-mon[63880]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:02:35.057 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:35 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/2549635654' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:02:35.310 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:35 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-activate[68476]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T12:02:35.311 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:35 smithi045 bash[68440]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T12:02:35.311 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:35 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-activate[68476]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T12:02:35.311 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:35 smithi045 bash[68440]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T12:02:35.311 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:35 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-activate[68476]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-17T12:02:35.311 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:35 smithi045 bash[68440]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-17T12:02:35.311 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:35 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-activate[68476]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T12:02:35.311 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:35 smithi045 bash[68440]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T12:02:35.311 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:35 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-activate[68476]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-17T12:02:35.311 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:35 smithi045 bash[68440]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-17T12:02:35.311 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:35 smithi045 podman[68440]: 2024-09-17 12:02:35.186525598 +0000 UTC m=+3.000990054 container died 474097ded63b91d38edb5a40f98a68deeb6020ca970aeea792bb8fff129660e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-activate, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=main-7498493, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-17T12:02:35.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:35 smithi165 ceph-mon[49373]: from='client.44143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:02:35.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:35 smithi165 ceph-mon[49373]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:02:35.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:35 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/2549635654' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:02:35.690 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 228/1617 objects degraded (14.100%), 29 pgs degraded 2024-09-17T12:02:35.690 INFO:teuthology.orchestra.run.smithi045.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T12:02:35.690 INFO:teuthology.orchestra.run.smithi045.stdout: osd.0 (root=default,host=smithi045) is down 2024-09-17T12:02:35.690 INFO:teuthology.orchestra.run.smithi045.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 228/1617 objects degraded (14.100%), 29 pgs degraded 2024-09-17T12:02:35.690 INFO:teuthology.orchestra.run.smithi045.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-09-17T12:02:35.690 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-09-17T12:02:35.690 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-09-17T12:02:35.690 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-09-17T12:02:35.690 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.5 is active+undersized+degraded, acting [7,4] 2024-09-17T12:02:35.691 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-09-17T12:02:35.691 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-09-17T12:02:35.691 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.f is active+undersized+degraded, acting [4,7] 2024-09-17T12:02:35.691 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-09-17T12:02:35.691 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-09-17T12:02:35.691 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-09-17T12:02:35.691 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-09-17T12:02:35.691 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-09-17T12:02:35.691 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-09-17T12:02:35.691 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.1f is active+undersized+degraded, acting [2,4] 2024-09-17T12:02:35.691 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-09-17T12:02:35.691 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-09-17T12:02:35.692 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-09-17T12:02:35.692 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-09-17T12:02:35.692 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-09-17T12:02:35.692 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-09-17T12:02:35.692 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-09-17T12:02:35.692 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-09-17T12:02:35.692 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-09-17T12:02:35.692 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-09-17T12:02:35.692 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-09-17T12:02:35.692 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-09-17T12:02:35.692 INFO:teuthology.orchestra.run.smithi045.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-09-17T12:02:35.692 INFO:teuthology.orchestra.run.smithi045.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-09-17T12:02:35.743 INFO:teuthology.orchestra.run.smithi045.stdout: 5 151763 9.75 MB/sec execute 276 sec latency 35.094 ms 2024-09-17T12:02:36.064 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:36 smithi045 ceph-mon[63880]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.9 MiB/s wr, 564 op/s; 228/1617 objects degraded (14.100%) 2024-09-17T12:02:36.064 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:36 smithi045 ceph-mon[63880]: from='client.34216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:02:36.064 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:36 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/1039138338' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:02:36.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:36 smithi165 ceph-mon[49373]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.9 MiB/s wr, 564 op/s; 228/1617 objects degraded (14.100%) 2024-09-17T12:02:36.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:36 smithi165 ceph-mon[49373]: from='client.34216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:02:36.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:36 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/1039138338' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:02:36.567 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:36 smithi045 podman[68720]: 2024-09-17 12:02:36.474116766 +0000 UTC m=+1.278986444 container remove 474097ded63b91d38edb5a40f98a68deeb6020ca970aeea792bb8fff129660e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-activate, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-17T12:02:36.743 INFO:teuthology.orchestra.run.smithi045.stdout: 5 152240 9.77 MB/sec execute 277 sec latency 35.077 ms 2024-09-17T12:02:36.822 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:36 smithi045 podman[68797]: 2024-09-17 12:02:36.630791641 +0000 UTC m=+0.024021821 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:02:37.079 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:36 smithi045 podman[68797]: 2024-09-17 12:02:36.958961246 +0000 UTC m=+0.352191416 container create 20dafd01bd124690d2f84a35f7d611f5ed67c6a13a79002077b2475e976ead9d (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=main-7498493, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-17T12:02:37.582 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:37 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:37.582 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:37 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:02:37.582 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:37 smithi045 ceph-mon[63880]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.7 MiB/s wr, 257 op/s; 228/1617 objects degraded (14.100%) 2024-09-17T12:02:37.582 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:37 smithi045 podman[68797]: 2024-09-17 12:02:37.408967677 +0000 UTC m=+0.802197859 container init 20dafd01bd124690d2f84a35f7d611f5ed67c6a13a79002077b2475e976ead9d (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=main-7498493, GIT_CLEAN=True) 2024-09-17T12:02:37.583 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:37 smithi045 podman[68797]: 2024-09-17 12:02:37.414236575 +0000 UTC m=+0.807466753 container start 20dafd01bd124690d2f84a35f7d611f5ed67c6a13a79002077b2475e976ead9d (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, RELEASE=main-7498493, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-17T12:02:37.583 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:37 smithi045 ceph-osd[68814]: -- 172.21.15.45:0/280398447 <== mon.0 v2:172.21.15.45:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55bf251081e0 con 0x55bf242ffc00 2024-09-17T12:02:37.743 INFO:teuthology.orchestra.run.smithi045.stdout: 5 152606 9.78 MB/sec execute 278 sec latency 49.964 ms 2024-09-17T12:02:37.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:37 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:37.854 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:37 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:02:37.855 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:37 smithi165 ceph-mon[49373]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.7 MiB/s wr, 257 op/s; 228/1617 objects degraded (14.100%) 2024-09-17T12:02:37.855 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:37 smithi045 bash[68797]: 20dafd01bd124690d2f84a35f7d611f5ed67c6a13a79002077b2475e976ead9d 2024-09-17T12:02:37.855 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:37 smithi045 systemd[1]: Started Ceph osd.0 for a12537de-74ea-11ef-bceb-c7b262605968. 2024-09-17T12:02:38.743 INFO:teuthology.orchestra.run.smithi045.stdout: 5 152934 9.77 MB/sec execute 279 sec latency 102.787 ms 2024-09-17T12:02:38.847 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:38 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:38.847 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:38 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:38.847 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:38 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:02:39.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:38 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:39.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:38 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:39.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:38 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:02:39.364 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:39 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0[68810]: 2024-09-17T12:02:39.266+0000 7f2f52d44640 -1 Falling back to public interface 2024-09-17T12:02:39.743 INFO:teuthology.orchestra.run.smithi045.stdout: 5 153339 9.76 MB/sec execute 280 sec latency 63.031 ms 2024-09-17T12:02:39.877 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:39 smithi045 ceph-mon[63880]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.8 MiB/s wr, 565 op/s; 232/1686 objects degraded (13.760%) 2024-09-17T12:02:40.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:39 smithi165 ceph-mon[49373]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.8 MiB/s wr, 565 op/s; 232/1686 objects degraded (13.760%) 2024-09-17T12:02:40.743 INFO:teuthology.orchestra.run.smithi045.stdout: 5 153738 9.77 MB/sec execute 281 sec latency 33.929 ms 2024-09-17T12:02:40.885 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:40 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 232/1686 objects degraded (13.760%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T12:02:41.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:40 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 232/1686 objects degraded (13.760%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T12:02:41.743 INFO:teuthology.orchestra.run.smithi045.stdout: 5 154121 9.78 MB/sec execute 282 sec latency 37.337 ms 2024-09-17T12:02:41.901 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:41 smithi045 ceph-mon[63880]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.9 MiB/s wr, 476 op/s; 232/1686 objects degraded (13.760%) 2024-09-17T12:02:42.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:41 smithi165 ceph-mon[49373]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.9 MiB/s wr, 476 op/s; 232/1686 objects degraded (13.760%) 2024-09-17T12:02:42.743 INFO:teuthology.orchestra.run.smithi045.stdout: 5 154716 9.79 MB/sec execute 283 sec latency 53.724 ms 2024-09-17T12:02:43.743 INFO:teuthology.orchestra.run.smithi045.stdout: 5 155117 9.79 MB/sec execute 284 sec latency 33.053 ms 2024-09-17T12:02:43.945 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:43 smithi045 ceph-mon[63880]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.1 MiB/s wr, 702 op/s; 232/1653 objects degraded (14.035%) 2024-09-17T12:02:44.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:43 smithi165 ceph-mon[49373]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.1 MiB/s wr, 702 op/s; 232/1653 objects degraded (14.035%) 2024-09-17T12:02:44.745 INFO:teuthology.orchestra.run.smithi045.stdout: 5 155497 9.81 MB/sec execute 285 sec latency 62.750 ms 2024-09-17T12:02:45.745 INFO:teuthology.orchestra.run.smithi045.stdout: 5 155850 9.80 MB/sec execute 286 sec latency 44.138 ms 2024-09-17T12:02:45.959 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:45 smithi045 ceph-mon[63880]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 511 op/s; 232/1653 objects degraded (14.035%) 2024-09-17T12:02:46.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:45 smithi165 ceph-mon[49373]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 511 op/s; 232/1653 objects degraded (14.035%) 2024-09-17T12:02:46.514 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:46 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0[68810]: 2024-09-17T12:02:46.386+0000 7f2f52d44640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-17T12:02:46.745 INFO:teuthology.orchestra.run.smithi045.stdout: 5 156102 9.79 MB/sec execute 287 sec latency 77.080 ms 2024-09-17T12:02:47.459 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:47 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0[68810]: 2024-09-17T12:02:47.426+0000 7f2f52d44640 -1 osd.0 72 log_to_monitors true 2024-09-17T12:02:47.745 INFO:teuthology.orchestra.run.smithi045.stdout: 5 156370 9.79 MB/sec execute 288 sec latency 54.769 ms 2024-09-17T12:02:48.148 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:47 smithi045 ceph-mon[63880]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 511 op/s; 232/1653 objects degraded (14.035%) 2024-09-17T12:02:48.148 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:47 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:48.148 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:47 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:48.148 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:47 smithi045 ceph-mon[63880]: from='osd.0 [v2:172.21.15.45:6802/713756418,v1:172.21.15.45:6803/713756418]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-17T12:02:48.148 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:47 smithi045 ceph-mon[63880]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-17T12:02:48.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:47 smithi165 ceph-mon[49373]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 511 op/s; 232/1653 objects degraded (14.035%) 2024-09-17T12:02:48.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:47 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:48.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:47 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:48.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:47 smithi165 ceph-mon[49373]: from='osd.0 [v2:172.21.15.45:6802/713756418,v1:172.21.15.45:6803/713756418]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-17T12:02:48.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:47 smithi165 ceph-mon[49373]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-17T12:02:48.661 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:02:48 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0[68810]: 2024-09-17T12:02:48.555+0000 7f2f46ae6640 -1 osd.0 72 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T12:02:48.745 INFO:teuthology.orchestra.run.smithi045.stdout: 5 156677 9.78 MB/sec execute 289 sec latency 82.545 ms 2024-09-17T12:02:49.164 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:48 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:49.165 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:48 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:49.165 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:48 smithi045 ceph-mon[63880]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T12:02:49.165 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:48 smithi045 ceph-mon[63880]: osdmap e75: 8 total, 7 up, 8 in 2024-09-17T12:02:49.165 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:48 smithi045 ceph-mon[63880]: from='osd.0 [v2:172.21.15.45:6802/713756418,v1:172.21.15.45:6803/713756418]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi045", "root=default"]} : dispatch 2024-09-17T12:02:49.165 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:48 smithi045 ceph-mon[63880]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi045", "root=default"]} : dispatch 2024-09-17T12:02:49.165 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:48 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 232/1653 objects degraded (14.035%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T12:02:49.165 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:48 smithi045 ceph-mon[63880]: from='osd.0 ' entity='osd.0' 2024-09-17T12:02:49.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:48 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:49.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:48 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:49.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:48 smithi165 ceph-mon[49373]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T12:02:49.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:48 smithi165 ceph-mon[49373]: osdmap e75: 8 total, 7 up, 8 in 2024-09-17T12:02:49.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:48 smithi165 ceph-mon[49373]: from='osd.0 [v2:172.21.15.45:6802/713756418,v1:172.21.15.45:6803/713756418]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi045", "root=default"]} : dispatch 2024-09-17T12:02:49.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:48 smithi165 ceph-mon[49373]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi045", "root=default"]} : dispatch 2024-09-17T12:02:49.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:48 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 232/1653 objects degraded (14.035%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T12:02:49.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:48 smithi165 ceph-mon[49373]: from='osd.0 ' entity='osd.0' 2024-09-17T12:02:49.745 INFO:teuthology.orchestra.run.smithi045.stdout: 5 156957 9.77 MB/sec execute 290 sec latency 248.109 ms 2024-09-17T12:02:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:50 smithi165 ceph-mon[49373]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.7 MiB/s wr, 558 op/s; 227/1590 objects degraded (14.277%) 2024-09-17T12:02:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:50 smithi165 ceph-mon[49373]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T12:02:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:50 smithi165 ceph-mon[49373]: osd.0 [v2:172.21.15.45:6802/713756418,v1:172.21.15.45:6803/713756418] boot 2024-09-17T12:02:50.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:50 smithi165 ceph-mon[49373]: osdmap e76: 8 total, 8 up, 8 in 2024-09-17T12:02:50.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:50 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-17T12:02:50.337 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:49 smithi045 ceph-mon[63880]: pgmap v65: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.7 MiB/s wr, 558 op/s; 227/1590 objects degraded (14.277%) 2024-09-17T12:02:50.337 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:49 smithi045 ceph-mon[63880]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T12:02:50.337 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:49 smithi045 ceph-mon[63880]: osd.0 [v2:172.21.15.45:6802/713756418,v1:172.21.15.45:6803/713756418] boot 2024-09-17T12:02:50.337 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:49 smithi045 ceph-mon[63880]: osdmap e76: 8 total, 8 up, 8 in 2024-09-17T12:02:50.337 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:49 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-17T12:02:50.745 INFO:teuthology.orchestra.run.smithi045.stdout: 5 156957 9.73 MB/sec execute 291 sec latency 1248.140 ms 2024-09-17T12:02:51.745 INFO:teuthology.orchestra.run.smithi045.stdout: 5 156978 9.70 MB/sec execute 292 sec latency 2142.565 ms 2024-09-17T12:02:51.778 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:51 smithi045 ceph-mon[63880]: osdmap e77: 8 total, 8 up, 8 in 2024-09-17T12:02:51.778 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:51 smithi045 ceph-mon[63880]: pgmap v68: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.9 MiB/s wr, 468 op/s; 227/1590 objects degraded (14.277%) 2024-09-17T12:02:51.779 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:51.779 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:02:52.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:51 smithi165 ceph-mon[49373]: osdmap e77: 8 total, 8 up, 8 in 2024-09-17T12:02:52.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:51 smithi165 ceph-mon[49373]: pgmap v68: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.9 MiB/s wr, 468 op/s; 227/1590 objects degraded (14.277%) 2024-09-17T12:02:52.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:51 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:52.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:51 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:02:52.542 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:52 smithi045 ceph-mon[63880]: osdmap e78: 8 total, 8 up, 8 in 2024-09-17T12:02:52.745 INFO:teuthology.orchestra.run.smithi045.stdout: 5 157285 9.70 MB/sec execute 293 sec latency 209.392 ms 2024-09-17T12:02:53.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:52 smithi165 ceph-mon[49373]: osdmap e78: 8 total, 8 up, 8 in 2024-09-17T12:02:53.556 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:53 smithi045 ceph-mon[63880]: osdmap e79: 8 total, 8 up, 8 in 2024-09-17T12:02:53.556 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:53 smithi045 ceph-mon[63880]: pgmap v71: 97 pgs: 11 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+remapped, 5 active+undersized, 2 active+undersized+degraded, 74 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.3 MiB/s wr, 331 op/s; 280/1476 objects degraded (18.970%); 3.8 MiB/s, 7 objects/s recovering 2024-09-17T12:02:53.556 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:53 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 227/1590 objects degraded (14.277%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T12:02:53.557 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:53 smithi045 ceph-mon[63880]: osdmap e80: 8 total, 8 up, 8 in 2024-09-17T12:02:53.557 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:53 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:53.557 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:53 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:53.557 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:53 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:02:53.557 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:53 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:02:53.557 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:53 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:53.557 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:53 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:53.745 INFO:teuthology.orchestra.run.smithi045.stdout: 5 157581 9.69 MB/sec execute 294 sec latency 679.203 ms 2024-09-17T12:02:54.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:53 smithi165 ceph-mon[49373]: osdmap e79: 8 total, 8 up, 8 in 2024-09-17T12:02:54.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:53 smithi165 ceph-mon[49373]: pgmap v71: 97 pgs: 11 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+remapped, 5 active+undersized, 2 active+undersized+degraded, 74 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.3 MiB/s wr, 331 op/s; 280/1476 objects degraded (18.970%); 3.8 MiB/s, 7 objects/s recovering 2024-09-17T12:02:54.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:53 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 227/1590 objects degraded (14.277%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T12:02:54.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:53 smithi165 ceph-mon[49373]: osdmap e80: 8 total, 8 up, 8 in 2024-09-17T12:02:54.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:53 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:54.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:53 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:54.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:53 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:02:54.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:53 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:02:54.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:53 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:54.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:53 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:02:54.570 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:54 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:02:54.570 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:54 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:02:54.570 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:54 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:02:54.570 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:54 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:02:54.570 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:54 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-17T12:02:54.570 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:54 smithi045 ceph-mon[63880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T12:02:54.570 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:54 smithi045 ceph-mon[63880]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-09-17T12:02:54.571 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:54 smithi045 ceph-mon[63880]: osdmap e81: 8 total, 8 up, 8 in 2024-09-17T12:02:54.745 INFO:teuthology.orchestra.run.smithi045.stdout: 5 157886 9.68 MB/sec execute 295 sec latency 925.992 ms 2024-09-17T12:02:55.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:54 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:02:55.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:54 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:02:55.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:54 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:02:55.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:54 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:02:55.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:54 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-17T12:02:55.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:54 smithi165 ceph-mon[49373]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T12:02:55.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:54 smithi165 ceph-mon[49373]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-09-17T12:02:55.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:54 smithi165 ceph-mon[49373]: osdmap e81: 8 total, 8 up, 8 in 2024-09-17T12:02:55.730 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:55 smithi045 ceph-mon[63880]: pgmap v74: 97 pgs: 2 peering, 11 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 81 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.4 MiB/s wr, 338 op/s; 262/1485 objects degraded (17.643%); 2.8 MiB/s, 12 objects/s recovering 2024-09-17T12:02:55.730 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:55 smithi045 ceph-mon[63880]: osdmap e82: 8 total, 8 up, 8 in 2024-09-17T12:02:55.745 INFO:teuthology.orchestra.run.smithi045.stdout: 5 158026 9.66 MB/sec execute 296 sec latency 756.726 ms 2024-09-17T12:02:56.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:55 smithi165 ceph-mon[49373]: pgmap v74: 97 pgs: 2 peering, 11 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 81 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.4 MiB/s wr, 338 op/s; 262/1485 objects degraded (17.643%); 2.8 MiB/s, 12 objects/s recovering 2024-09-17T12:02:56.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:55 smithi165 ceph-mon[49373]: osdmap e82: 8 total, 8 up, 8 in 2024-09-17T12:02:56.745 INFO:teuthology.orchestra.run.smithi045.stdout: 5 158275 9.65 MB/sec execute 297 sec latency 926.365 ms 2024-09-17T12:02:57.266 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:57 smithi045 ceph-mon[63880]: osdmap e83: 8 total, 8 up, 8 in 2024-09-17T12:02:57.266 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:57 smithi045 ceph-mon[63880]: pgmap v77: 97 pgs: 2 peering, 11 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 81 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 69 KiB/s wr, 7 op/s; 262/1485 objects degraded (17.643%); 0 B/s, 4 objects/s recovering 2024-09-17T12:02:57.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:57 smithi165 ceph-mon[49373]: osdmap e83: 8 total, 8 up, 8 in 2024-09-17T12:02:57.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:57 smithi165 ceph-mon[49373]: pgmap v77: 97 pgs: 2 peering, 11 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 81 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 69 KiB/s wr, 7 op/s; 262/1485 objects degraded (17.643%); 0 B/s, 4 objects/s recovering 2024-09-17T12:02:57.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 158579 9.65 MB/sec execute 298 sec latency 358.853 ms 2024-09-17T12:02:58.293 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:58 smithi045 ceph-mon[63880]: osdmap e84: 8 total, 8 up, 8 in 2024-09-17T12:02:58.293 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:58 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 262/1485 objects degraded (17.643%), 11 pgs degraded (PG_DEGRADED) 2024-09-17T12:02:58.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:58 smithi165 ceph-mon[49373]: osdmap e84: 8 total, 8 up, 8 in 2024-09-17T12:02:58.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:58 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 262/1485 objects degraded (17.643%), 11 pgs degraded (PG_DEGRADED) 2024-09-17T12:02:58.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 158928 9.64 MB/sec execute 299 sec latency 55.249 ms 2024-09-17T12:02:59.317 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:02:59 smithi045 ceph-mon[63880]: pgmap v79: 97 pgs: 2 peering, 95 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.8 MiB/s wr, 427 op/s; 2.5 MiB/s, 56 objects/s recovering 2024-09-17T12:02:59.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:02:59 smithi165 ceph-mon[49373]: pgmap v79: 97 pgs: 2 peering, 95 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.8 MiB/s wr, 427 op/s; 2.5 MiB/s, 56 objects/s recovering 2024-09-17T12:02:59.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 159160 9.63 MB/sec execute 300 sec latency 58.028 ms 2024-09-17T12:03:00.344 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:00 smithi045 ceph-mon[63880]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 262/1485 objects degraded (17.643%), 11 pgs degraded) 2024-09-17T12:03:00.344 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:00 smithi045 ceph-mon[63880]: Cluster is now healthy 2024-09-17T12:03:00.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:00 smithi165 ceph-mon[49373]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 262/1485 objects degraded (17.643%), 11 pgs degraded) 2024-09-17T12:03:00.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:00 smithi165 ceph-mon[49373]: Cluster is now healthy 2024-09-17T12:03:00.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 159405 9.63 MB/sec execute 301 sec latency 59.021 ms 2024-09-17T12:03:01.375 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:01 smithi045 ceph-mon[63880]: pgmap v80: 97 pgs: 2 peering, 95 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 2.9 MiB/s wr, 331 op/s; 2.0 MiB/s, 43 objects/s recovering 2024-09-17T12:03:01.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:01 smithi165 ceph-mon[49373]: pgmap v80: 97 pgs: 2 peering, 95 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 2.9 MiB/s wr, 331 op/s; 2.0 MiB/s, 43 objects/s recovering 2024-09-17T12:03:01.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 160018 9.67 MB/sec execute 302 sec latency 41.131 ms 2024-09-17T12:03:02.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 160740 9.69 MB/sec execute 303 sec latency 21.593 ms 2024-09-17T12:03:03.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 161403 9.71 MB/sec execute 304 sec latency 17.285 ms 2024-09-17T12:03:03.920 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:03 smithi045 ceph-mon[63880]: pgmap v81: 97 pgs: 97 active+clean; 350 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.5 MiB/s wr, 563 op/s; 1.5 MiB/s, 34 objects/s recovering 2024-09-17T12:03:04.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:03 smithi165 ceph-mon[49373]: pgmap v81: 97 pgs: 97 active+clean; 350 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.5 MiB/s wr, 563 op/s; 1.5 MiB/s, 34 objects/s recovering 2024-09-17T12:03:04.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 162082 9.74 MB/sec execute 305 sec latency 23.947 ms 2024-09-17T12:03:05.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 162473 9.73 MB/sec execute 306 sec latency 86.320 ms 2024-09-17T12:03:06.159 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:05 smithi045 ceph-mon[63880]: pgmap v82: 97 pgs: 97 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.5 MiB/s wr, 521 op/s; 1.4 MiB/s, 30 objects/s recovering 2024-09-17T12:03:06.224 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T12:03:06.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:05 smithi165 ceph-mon[49373]: pgmap v82: 97 pgs: 97 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.5 MiB/s wr, 521 op/s; 1.4 MiB/s, 30 objects/s recovering 2024-09-17T12:03:06.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 162780 9.73 MB/sec execute 307 sec latency 59.111 ms 2024-09-17T12:03:07.220 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T12:03:07.220 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (4m) 20s ago 13m 20.4M - 0.27.0 11f11916f8cd 2a2a54402d2a 2024-09-17T12:03:07.220 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (68s) 20s ago 13m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 e545cbf7736d 2024-09-17T12:03:07.221 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (63s) 58s ago 13m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 c1e89e114e4d 2024-09-17T12:03:07.221 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (4m) 20s ago 13m 58.3M - dad864ee21e9 371aaf55379f 2024-09-17T12:03:07.221 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (7m) 20s ago 7m 37.3M - 17.2.0 e1d6a67b021e 6a2a4a998dd1 2024-09-17T12:03:07.221 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (7m) 58s ago 7m 16.0M - 17.2.0 e1d6a67b021e d4c2987b9bac 2024-09-17T12:03:07.221 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:8443,9283,8765 running (3m) 20s ago 14m 513M - 19.3.0-4937-g74984932 6c1fcab85602 ae2f507bd67d 2024-09-17T12:03:07.221 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283,8765 running (2m) 58s ago 13m 453M - 19.3.0-4937-g74984932 6c1fcab85602 53403866652e 2024-09-17T12:03:07.221 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (2m) 20s ago 15m 48.5M 2048M 19.3.0-4937-g74984932 6c1fcab85602 737eec78e0f0 2024-09-17T12:03:07.221 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (118s) 58s ago 12m 36.1M 2048M 19.3.0-4937-g74984932 6c1fcab85602 249eedb4123e 2024-09-17T12:03:07.221 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (7m) 20s ago 7m 261M - 4.0 e1d6a67b021e b8fc749ebb71 2024-09-17T12:03:07.221 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (7m) 58s ago 7m 83.8M - 4.0 e1d6a67b021e 771b27764432 2024-09-17T12:03:07.221 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (4m) 20s ago 13m 11.3M - 1.7.0 72c9c2088986 989b5738cb69 2024-09-17T12:03:07.221 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (3m) 58s ago 12m 11.9M - 1.7.0 72c9c2088986 e1e8ffab3f21 2024-09-17T12:03:07.221 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (30s) 20s ago 12m 11.1M 1937M 19.3.0-4937-g74984932 6c1fcab85602 20dafd01bd12 2024-09-17T12:03:07.222 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (11m) 20s ago 11m 476M 1937M 17.2.0 e1d6a67b021e da8bda7b47b3 2024-09-17T12:03:07.222 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (11m) 20s ago 11m 347M 1937M 17.2.0 e1d6a67b021e 21fe0d9e5ced 2024-09-17T12:03:07.222 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (10m) 20s ago 10m 497M 1937M 17.2.0 e1d6a67b021e 92a2389488e4 2024-09-17T12:03:07.222 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (10m) 58s ago 10m 494M 2704M 17.2.0 e1d6a67b021e e29eb74df25a 2024-09-17T12:03:07.222 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (10m) 58s ago 10m 544M 2704M 17.2.0 e1d6a67b021e 5d900eb9165b 2024-09-17T12:03:07.222 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (9m) 58s ago 9m 466M 2704M 17.2.0 e1d6a67b021e fae3e1bae466 2024-09-17T12:03:07.222 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (9m) 58s ago 9m 371M 2704M 17.2.0 e1d6a67b021e 71b9f2cdb810 2024-09-17T12:03:07.222 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (2m) 20s ago 12m 53.6M - 2.51.0 1d3b7f56885b 61b064ad6609 2024-09-17T12:03:07.476 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:07 smithi045 ceph-mon[63880]: from='client.34228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:03:07.476 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:07 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:07.476 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:07 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:03:07.476 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:07 smithi045 ceph-mon[63880]: from='client.34232 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:03:07.476 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:07 smithi045 ceph-mon[63880]: pgmap v83: 97 pgs: 97 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 5.1 MiB/s wr, 469 op/s; 1.2 MiB/s, 26 objects/s recovering 2024-09-17T12:03:07.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 163051 9.73 MB/sec execute 308 sec latency 54.139 ms 2024-09-17T12:03:07.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:07 smithi165 ceph-mon[49373]: from='client.34228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:03:07.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:07 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:07.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:07 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:03:07.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:07 smithi165 ceph-mon[49373]: from='client.34232 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:03:07.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:07 smithi165 ceph-mon[49373]: pgmap v83: 97 pgs: 97 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 5.1 MiB/s wr, 469 op/s; 1.2 MiB/s, 26 objects/s recovering 2024-09-17T12:03:07.818 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:03:07.818 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T12:03:07.818 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:03:07.818 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:03:07.818 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T12:03:07.819 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:03:07.819 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:03:07.819 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T12:03:07.819 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-17T12:03:07.819 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 1 2024-09-17T12:03:07.819 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:03:07.819 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T12:03:07.819 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T12:03:07.819 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:03:07.819 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T12:03:07.819 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-17T12:03:07.819 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 5 2024-09-17T12:03:07.819 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T12:03:07.819 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:03:08.313 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:03:08.313 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T12:03:08.313 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T12:03:08.313 INFO:teuthology.orchestra.run.smithi045.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T12:03:08.313 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [ 2024-09-17T12:03:08.313 INFO:teuthology.orchestra.run.smithi045.stdout: "crash", 2024-09-17T12:03:08.313 INFO:teuthology.orchestra.run.smithi045.stdout: "mon", 2024-09-17T12:03:08.313 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr" 2024-09-17T12:03:08.313 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-17T12:03:08.313 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "7/23 daemons upgraded", 2024-09-17T12:03:08.313 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T12:03:08.313 INFO:teuthology.orchestra.run.smithi045.stdout: "is_paused": false 2024-09-17T12:03:08.314 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:03:08.495 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:08 smithi045 ceph-mon[63880]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:03:08.495 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:08 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/4045942867' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:03:08.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 163366 9.74 MB/sec execute 309 sec latency 46.114 ms 2024-09-17T12:03:08.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:08 smithi165 ceph-mon[49373]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:03:08.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:08 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/4045942867' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:03:08.975 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_OK 2024-09-17T12:03:09.512 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:09 smithi045 ceph-mon[63880]: from='client.34244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:03:09.512 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:09 smithi045 ceph-mon[63880]: pgmap v84: 97 pgs: 97 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.8 MiB/s wr, 560 op/s; 352 KiB/s, 8 objects/s recovering 2024-09-17T12:03:09.512 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:09 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-17T12:03:09.512 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:09 smithi045 ceph-mon[63880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T12:03:09.512 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:09 smithi045 ceph-mon[63880]: Upgrade: osd.1 is safe to restart 2024-09-17T12:03:09.512 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:09 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/2204225854' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:03:09.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 163688 9.74 MB/sec execute 310 sec latency 52.700 ms 2024-09-17T12:03:09.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:09 smithi165 ceph-mon[49373]: from='client.34244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:03:09.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:09 smithi165 ceph-mon[49373]: pgmap v84: 97 pgs: 97 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.8 MiB/s wr, 560 op/s; 352 KiB/s, 8 objects/s recovering 2024-09-17T12:03:09.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:09 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-17T12:03:09.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:09 smithi165 ceph-mon[49373]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T12:03:09.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:09 smithi165 ceph-mon[49373]: Upgrade: osd.1 is safe to restart 2024-09-17T12:03:09.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:09 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/2204225854' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:03:10.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 163969 9.73 MB/sec execute 311 sec latency 48.804 ms 2024-09-17T12:03:11.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 164202 9.71 MB/sec execute 312 sec latency 46.652 ms 2024-09-17T12:03:12.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:11 smithi045 ceph-mon[63880]: pgmap v85: 97 pgs: 97 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.5 MiB/s wr, 530 op/s 2024-09-17T12:03:12.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:11 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:12.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:11 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:12.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:11 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-17T12:03:12.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:11 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:03:12.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:11 smithi165 ceph-mon[49373]: pgmap v85: 97 pgs: 97 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.5 MiB/s wr, 530 op/s 2024-09-17T12:03:12.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:11 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:12.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:11 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:12.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:11 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-17T12:03:12.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:11 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:03:12.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 164464 9.69 MB/sec execute 313 sec latency 59.350 ms 2024-09-17T12:03:13.096 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:12 smithi045 ceph-mon[63880]: Upgrade: Updating osd.1 2024-09-17T12:03:13.097 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:12 smithi045 ceph-mon[63880]: Deploying daemon osd.1 on smithi045 2024-09-17T12:03:13.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:12 smithi165 ceph-mon[49373]: Upgrade: Updating osd.1 2024-09-17T12:03:13.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:12 smithi165 ceph-mon[49373]: Deploying daemon osd.1 on smithi045 2024-09-17T12:03:13.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 164818 9.69 MB/sec execute 314 sec latency 49.620 ms 2024-09-17T12:03:14.121 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:13 smithi045 ceph-mon[63880]: pgmap v86: 97 pgs: 97 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 7.2 MiB/s wr, 693 op/s 2024-09-17T12:03:14.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:13 smithi165 ceph-mon[49373]: pgmap v86: 97 pgs: 97 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 7.2 MiB/s wr, 693 op/s 2024-09-17T12:03:14.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 165130 9.68 MB/sec execute 315 sec latency 49.978 ms 2024-09-17T12:03:15.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 165571 9.70 MB/sec execute 316 sec latency 41.966 ms 2024-09-17T12:03:16.155 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:15 smithi045 ceph-mon[63880]: pgmap v87: 97 pgs: 97 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.4 MiB/s wr, 511 op/s 2024-09-17T12:03:16.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:15 smithi165 ceph-mon[49373]: pgmap v87: 97 pgs: 97 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.4 MiB/s wr, 511 op/s 2024-09-17T12:03:16.407 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:16 smithi045 systemd[1]: Stopping Ceph osd.1 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:03:16.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 165911 9.69 MB/sec execute 317 sec latency 50.199 ms 2024-09-17T12:03:16.912 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:16 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1[37003]: 2024-09-17T12:03:16.666+0000 7f5f048e6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T12:03:16.912 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:16 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1[37003]: 2024-09-17T12:03:16.666+0000 7f5f048e6700 -1 osd.1 84 *** Got signal Terminated *** 2024-09-17T12:03:16.912 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:16 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1[37003]: 2024-09-17T12:03:16.666+0000 7f5f048e6700 -1 osd.1 84 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T12:03:17.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:16 smithi165 ceph-mon[49373]: osd.1 marked itself down and dead 2024-09-17T12:03:17.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:16 smithi045 ceph-mon[63880]: osd.1 marked itself down and dead 2024-09-17T12:03:17.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 165969 9.66 MB/sec execute 318 sec latency 816.661 ms 2024-09-17T12:03:17.839 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:17 smithi045 podman[70696]: 2024-09-17 12:03:17.577923791 +0000 UTC m=+1.049007207 container died da8bda7b47b3705274b07895c7beb975b57f8dbab31d0fd9940ab590f885c284 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1, ceph=True, name=centos-stream, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, io.openshift.expose-services=, release=754, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64) 2024-09-17T12:03:18.184 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:17 smithi045 ceph-mon[63880]: pgmap v88: 97 pgs: 97 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 503 op/s 2024-09-17T12:03:18.185 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:17 smithi045 ceph-mon[63880]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T12:03:18.185 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:17 smithi045 ceph-mon[63880]: osdmap e85: 8 total, 7 up, 8 in 2024-09-17T12:03:18.185 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:17 smithi045 podman[70696]: 2024-09-17 12:03:17.932716988 +0000 UTC m=+1.403800395 container cleanup da8bda7b47b3705274b07895c7beb975b57f8dbab31d0fd9940ab590f885c284 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1, ceph=True, distribution-scope=public, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, RELEASE=HEAD, com.redhat.component=centos-stream-container, name=centos-stream, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , io.openshift.expose-services=, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git) 2024-09-17T12:03:18.185 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:17 smithi045 bash[70696]: ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1 2024-09-17T12:03:18.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:17 smithi165 ceph-mon[49373]: pgmap v88: 97 pgs: 97 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.0 MiB/s wr, 503 op/s 2024-09-17T12:03:18.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:17 smithi165 ceph-mon[49373]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T12:03:18.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:17 smithi165 ceph-mon[49373]: osdmap e85: 8 total, 7 up, 8 in 2024-09-17T12:03:18.437 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:18 smithi045 podman[70711]: 2024-09-17 12:03:18.356223857 +0000 UTC m=+0.778409263 container remove da8bda7b47b3705274b07895c7beb975b57f8dbab31d0fd9940ab590f885c284 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., name=centos-stream, vcs-type=git, RELEASE=HEAD, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-17T12:03:18.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 166319 9.66 MB/sec execute 319 sec latency 1038.350 ms 2024-09-17T12:03:18.944 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:18 smithi045 ceph-mon[63880]: osdmap e86: 8 total, 7 up, 8 in 2024-09-17T12:03:18.945 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:18 smithi045 podman[70799]: 2024-09-17 12:03:18.766769734 +0000 UTC m=+0.021490942 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:03:19.195 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:19 smithi045 podman[70799]: 2024-09-17 12:03:19.023004141 +0000 UTC m=+0.277725328 container create 5b3e30c0e39d76f997ec4ee7de628add8b0055f40ada4149a8d0c8aa28f88735 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True) 2024-09-17T12:03:19.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:18 smithi165 ceph-mon[49373]: osdmap e86: 8 total, 7 up, 8 in 2024-09-17T12:03:19.452 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:19 smithi045 podman[70799]: 2024-09-17 12:03:19.433728455 +0000 UTC m=+0.688449652 container init 5b3e30c0e39d76f997ec4ee7de628add8b0055f40ada4149a8d0c8aa28f88735 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=main-7498493, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-17T12:03:19.452 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:19 smithi045 podman[70799]: 2024-09-17 12:03:19.442731902 +0000 UTC m=+0.697453086 container start 5b3e30c0e39d76f997ec4ee7de628add8b0055f40ada4149a8d0c8aa28f88735 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-deactivate, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, RELEASE=main-7498493, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-17T12:03:19.702 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:19 smithi045 podman[70799]: 2024-09-17 12:03:19.584130975 +0000 UTC m=+0.838852175 container attach 5b3e30c0e39d76f997ec4ee7de628add8b0055f40ada4149a8d0c8aa28f88735 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T12:03:19.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 166691 9.68 MB/sec execute 320 sec latency 27.971 ms 2024-09-17T12:03:19.959 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:19 smithi045 ceph-mon[63880]: pgmap v91: 97 pgs: 2 active+undersized, 5 peering, 9 stale+active+clean, 2 active+undersized+degraded, 79 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.7 MiB/s wr, 573 op/s; 18/1608 objects degraded (1.119%) 2024-09-17T12:03:19.959 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:19 smithi045 ceph-mon[63880]: Health check failed: Degraded data redundancy: 18/1608 objects degraded (1.119%), 2 pgs degraded (PG_DEGRADED) 2024-09-17T12:03:19.959 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:19 smithi045 podman[70799]: 2024-09-17 12:03:19.756713445 +0000 UTC m=+1.011434640 container died 5b3e30c0e39d76f997ec4ee7de628add8b0055f40ada4149a8d0c8aa28f88735 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, ceph=True, RELEASE=main-7498493, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T12:03:20.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:19 smithi165 ceph-mon[49373]: pgmap v91: 97 pgs: 2 active+undersized, 5 peering, 9 stale+active+clean, 2 active+undersized+degraded, 79 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.7 MiB/s wr, 573 op/s; 18/1608 objects degraded (1.119%) 2024-09-17T12:03:20.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:19 smithi165 ceph-mon[49373]: Health check failed: Degraded data redundancy: 18/1608 objects degraded (1.119%), 2 pgs degraded (PG_DEGRADED) 2024-09-17T12:03:20.747 INFO:teuthology.orchestra.run.smithi045.stdout: 5 167036 9.68 MB/sec execute 321 sec latency 43.730 ms 2024-09-17T12:03:20.981 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:20 smithi045 podman[70820]: 2024-09-17 12:03:20.949950602 +0000 UTC m=+1.186686273 container remove 5b3e30c0e39d76f997ec4ee7de628add8b0055f40ada4149a8d0c8aa28f88735 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=main-7498493, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True) 2024-09-17T12:03:21.236 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:21 smithi045 systemd[1]: ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-17T12:03:21.236 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:21 smithi045 systemd[1]: Stopped Ceph osd.1 for a12537de-74ea-11ef-bceb-c7b262605968. 2024-09-17T12:03:21.236 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:21 smithi045 systemd[1]: ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.1.service: Consumed 2min 46.053s CPU time. 2024-09-17T12:03:21.492 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:21 smithi045 systemd[1]: Starting Ceph osd.1 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:03:21.745 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:21 smithi045 podman[70918]: 2024-09-17 12:03:21.474315075 +0000 UTC m=+0.023489079 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:03:21.745 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:21 smithi045 podman[70918]: 2024-09-17 12:03:21.651925791 +0000 UTC m=+0.201099795 container create 34b47368237df6329d3f691806d6344b85a87503f4e7b20da630f07e9a69d5b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-activate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493) 2024-09-17T12:03:21.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 167331 9.68 MB/sec execute 322 sec latency 45.729 ms 2024-09-17T12:03:21.998 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:21 smithi045 ceph-mon[63880]: pgmap v92: 97 pgs: 2 active+undersized, 5 peering, 9 stale+active+clean, 2 active+undersized+degraded, 79 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.1 MiB/s wr, 330 op/s; 18/1608 objects degraded (1.119%) 2024-09-17T12:03:21.998 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:21 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:21.998 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:21 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:03:22.249 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:21 smithi045 podman[70918]: 2024-09-17 12:03:21.997079819 +0000 UTC m=+0.546253847 container init 34b47368237df6329d3f691806d6344b85a87503f4e7b20da630f07e9a69d5b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=main-7498493, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T12:03:22.249 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:22 smithi045 podman[70918]: 2024-09-17 12:03:22.007479751 +0000 UTC m=+0.556653763 container start 34b47368237df6329d3f691806d6344b85a87503f4e7b20da630f07e9a69d5b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, RELEASE=main-7498493, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T12:03:22.250 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:22 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-activate[70932]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:03:22.250 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:22 smithi045 podman[70918]: 2024-09-17 12:03:22.181106651 +0000 UTC m=+0.730280662 container attach 34b47368237df6329d3f691806d6344b85a87503f4e7b20da630f07e9a69d5b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-17T12:03:22.250 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:22 smithi045 bash[70918]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:03:22.250 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:22 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-activate[70932]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:03:22.250 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:22 smithi045 bash[70918]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:03:22.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:21 smithi165 ceph-mon[49373]: pgmap v92: 97 pgs: 2 active+undersized, 5 peering, 9 stale+active+clean, 2 active+undersized+degraded, 79 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.1 MiB/s wr, 330 op/s; 18/1608 objects degraded (1.119%) 2024-09-17T12:03:22.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:21 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:22.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:21 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:03:22.746 INFO:teuthology.orchestra.run.smithi045.stdout: 5 167577 9.67 MB/sec execute 323 sec latency 76.699 ms 2024-09-17T12:03:23.556 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:23 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-activate[70932]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T12:03:23.556 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:23 smithi045 bash[70918]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T12:03:23.556 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:23 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-activate[70932]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T12:03:23.556 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:23 smithi045 bash[70918]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T12:03:23.747 INFO:teuthology.orchestra.run.smithi045.stdout: 5 167931 9.66 MB/sec execute 324 sec latency 36.658 ms 2024-09-17T12:03:23.810 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:23 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-activate[70932]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T12:03:23.810 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:23 smithi045 bash[70918]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T12:03:23.810 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:23 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-activate[70932]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T12:03:23.810 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:23 smithi045 bash[70918]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T12:03:23.810 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:23 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-activate[70932]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-17T12:03:23.811 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:23 smithi045 bash[70918]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-17T12:03:23.811 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:23 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-activate[70932]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T12:03:23.811 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:23 smithi045 bash[70918]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T12:03:23.811 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:23 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-activate[70932]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-17T12:03:23.811 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:23 smithi045 bash[70918]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-17T12:03:24.061 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:23 smithi045 ceph-mon[63880]: pgmap v93: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 550 op/s; 191/1653 objects degraded (11.555%) 2024-09-17T12:03:24.061 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:23 smithi045 podman[70918]: 2024-09-17 12:03:23.809151182 +0000 UTC m=+2.358325191 container died 34b47368237df6329d3f691806d6344b85a87503f4e7b20da630f07e9a69d5b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=main-7498493, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-17T12:03:24.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:23 smithi165 ceph-mon[49373]: pgmap v93: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 550 op/s; 191/1653 objects degraded (11.555%) 2024-09-17T12:03:24.747 INFO:teuthology.orchestra.run.smithi045.stdout: 5 168302 9.65 MB/sec execute 325 sec latency 106.775 ms 2024-09-17T12:03:25.076 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:24 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 194/1668 objects degraded (11.631%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T12:03:25.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:24 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 194/1668 objects degraded (11.631%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T12:03:25.328 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:25 smithi045 podman[71084]: 2024-09-17 12:03:25.322519341 +0000 UTC m=+1.502921591 container remove 34b47368237df6329d3f691806d6344b85a87503f4e7b20da630f07e9a69d5b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=main-7498493, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-17T12:03:25.751 INFO:teuthology.orchestra.run.smithi045.stdout: 5 168637 9.65 MB/sec execute 326 sec latency 53.985 ms 2024-09-17T12:03:25.862 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:25 smithi045 podman[71138]: 2024-09-17 12:03:25.516880286 +0000 UTC m=+0.026953538 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:03:25.862 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:25 smithi045 podman[71138]: 2024-09-17 12:03:25.710154594 +0000 UTC m=+0.220227840 container create d39be5014418be8f139753fafa73dc2e93264a9d233b4eae1427d15abc47bd8f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True) 2024-09-17T12:03:26.114 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:25 smithi045 ceph-mon[63880]: pgmap v94: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.7 MiB/s wr, 559 op/s; 194/1668 objects degraded (11.631%) 2024-09-17T12:03:26.114 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:26 smithi045 podman[71138]: 2024-09-17 12:03:26.071285507 +0000 UTC m=+0.581358754 container init d39be5014418be8f139753fafa73dc2e93264a9d233b4eae1427d15abc47bd8f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=main-7498493, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T12:03:26.114 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:26 smithi045 podman[71138]: 2024-09-17 12:03:26.085882629 +0000 UTC m=+0.595955868 container start d39be5014418be8f139753fafa73dc2e93264a9d233b4eae1427d15abc47bd8f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1, RELEASE=main-7498493, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-17T12:03:26.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:25 smithi165 ceph-mon[49373]: pgmap v94: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.7 MiB/s wr, 559 op/s; 194/1668 objects degraded (11.631%) 2024-09-17T12:03:26.365 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:26 smithi045 ceph-osd[71156]: -- 172.21.15.45:0/1832276149 <== mon.1 v2:172.21.15.165:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a8dffaa1e0 con 0x55a8dff9a000 2024-09-17T12:03:26.365 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:26 smithi045 bash[71138]: d39be5014418be8f139753fafa73dc2e93264a9d233b4eae1427d15abc47bd8f 2024-09-17T12:03:26.365 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:26 smithi045 systemd[1]: Started Ceph osd.1 for a12537de-74ea-11ef-bceb-c7b262605968. 2024-09-17T12:03:26.752 INFO:teuthology.orchestra.run.smithi045.stdout: 5 169067 9.66 MB/sec execute 327 sec latency 54.725 ms 2024-09-17T12:03:27.394 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:27 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:27.394 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:27 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:27.394 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:27 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:03:27.394 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:27 smithi045 ceph-mon[63880]: pgmap v95: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.7 MiB/s wr, 534 op/s; 192/1650 objects degraded (11.636%) 2024-09-17T12:03:27.751 INFO:teuthology.orchestra.run.smithi045.stdout: 5 169501 9.66 MB/sec execute 328 sec latency 64.272 ms 2024-09-17T12:03:27.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:27 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:27.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:27 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:27.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:27 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:03:27.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:27 smithi165 ceph-mon[49373]: pgmap v95: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.7 MiB/s wr, 534 op/s; 192/1650 objects degraded (11.636%) 2024-09-17T12:03:28.586 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:28 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1[71152]: 2024-09-17T12:03:28.461+0000 7f76c9403640 -1 Falling back to public interface 2024-09-17T12:03:28.751 INFO:teuthology.orchestra.run.smithi045.stdout: 5 169891 9.66 MB/sec execute 329 sec latency 41.171 ms 2024-09-17T12:03:29.751 INFO:teuthology.orchestra.run.smithi045.stdout: 5 170233 9.67 MB/sec execute 330 sec latency 33.695 ms 2024-09-17T12:03:30.160 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:29 smithi045 ceph-mon[63880]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 443 op/s; 180/1593 objects degraded (11.299%) 2024-09-17T12:03:30.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:29 smithi165 ceph-mon[49373]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 443 op/s; 180/1593 objects degraded (11.299%) 2024-09-17T12:03:30.751 INFO:teuthology.orchestra.run.smithi045.stdout: 5 170541 9.67 MB/sec execute 331 sec latency 36.171 ms 2024-09-17T12:03:31.751 INFO:teuthology.orchestra.run.smithi045.stdout: 5 170854 9.68 MB/sec execute 332 sec latency 55.834 ms 2024-09-17T12:03:31.948 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:31 smithi045 ceph-mon[63880]: pgmap v97: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 392 op/s; 180/1593 objects degraded (11.299%) 2024-09-17T12:03:32.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:31 smithi165 ceph-mon[49373]: pgmap v97: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 342 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 392 op/s; 180/1593 objects degraded (11.299%) 2024-09-17T12:03:32.751 INFO:teuthology.orchestra.run.smithi045.stdout: 5 171249 9.67 MB/sec execute 333 sec latency 69.431 ms 2024-09-17T12:03:33.217 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:32 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 190/1659 objects degraded (11.453%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T12:03:33.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:32 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 190/1659 objects degraded (11.453%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T12:03:33.751 INFO:teuthology.orchestra.run.smithi045.stdout: 5 171755 9.67 MB/sec execute 334 sec latency 29.059 ms 2024-09-17T12:03:34.229 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:33 smithi045 ceph-mon[63880]: pgmap v98: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.1 MiB/s wr, 587 op/s; 190/1659 objects degraded (11.453%) 2024-09-17T12:03:34.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:33 smithi165 ceph-mon[49373]: pgmap v98: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.1 MiB/s wr, 587 op/s; 190/1659 objects degraded (11.453%) 2024-09-17T12:03:34.751 INFO:teuthology.orchestra.run.smithi045.stdout: 5 172219 9.68 MB/sec execute 335 sec latency 41.221 ms 2024-09-17T12:03:35.498 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:35 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1[71152]: 2024-09-17T12:03:35.338+0000 7f76c9403640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-17T12:03:35.751 INFO:teuthology.orchestra.run.smithi045.stdout: 5 172555 9.68 MB/sec execute 336 sec latency 57.202 ms 2024-09-17T12:03:36.010 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:35 smithi045 ceph-mon[63880]: pgmap v99: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.9 MiB/s wr, 484 op/s; 190/1665 objects degraded (11.411%) 2024-09-17T12:03:36.010 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:35 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:36.010 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:35 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:36.010 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:35 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:36.010 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:35 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:36.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:35 smithi165 ceph-mon[49373]: pgmap v99: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.9 MiB/s wr, 484 op/s; 190/1665 objects degraded (11.411%) 2024-09-17T12:03:36.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:35 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:36.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:35 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:36.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:35 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:36.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:35 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:36.515 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:36 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1[71152]: 2024-09-17T12:03:36.417+0000 7f76c9403640 -1 osd.1 84 log_to_monitors true 2024-09-17T12:03:36.751 INFO:teuthology.orchestra.run.smithi045.stdout: 5 172893 9.67 MB/sec execute 337 sec latency 60.208 ms 2024-09-17T12:03:37.032 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:03:37 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1[71152]: 2024-09-17T12:03:37.004+0000 7f76c11ad640 -1 osd.1 84 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T12:03:37.032 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:03:37.032 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:36 smithi045 ceph-mon[63880]: from='osd.1 [v2:172.21.15.45:6810/1032158886,v1:172.21.15.45:6811/1032158886]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-17T12:03:37.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:03:37.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:36 smithi165 ceph-mon[49373]: from='osd.1 [v2:172.21.15.45:6810/1032158886,v1:172.21.15.45:6811/1032158886]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-17T12:03:37.751 INFO:teuthology.orchestra.run.smithi045.stdout: 5 173301 9.68 MB/sec execute 338 sec latency 45.716 ms 2024-09-17T12:03:38.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:37 smithi045 ceph-mon[63880]: pgmap v100: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.8 MiB/s wr, 495 op/s; 184/1632 objects degraded (11.275%) 2024-09-17T12:03:38.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:37 smithi045 ceph-mon[63880]: from='osd.1 [v2:172.21.15.45:6810/1032158886,v1:172.21.15.45:6811/1032158886]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T12:03:38.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:37 smithi045 ceph-mon[63880]: osdmap e87: 8 total, 7 up, 8 in 2024-09-17T12:03:38.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:37 smithi045 ceph-mon[63880]: from='osd.1 [v2:172.21.15.45:6810/1032158886,v1:172.21.15.45:6811/1032158886]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi045", "root=default"]} : dispatch 2024-09-17T12:03:38.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:37 smithi165 ceph-mon[49373]: pgmap v100: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.8 MiB/s wr, 495 op/s; 184/1632 objects degraded (11.275%) 2024-09-17T12:03:38.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:37 smithi165 ceph-mon[49373]: from='osd.1 [v2:172.21.15.45:6810/1032158886,v1:172.21.15.45:6811/1032158886]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T12:03:38.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:37 smithi165 ceph-mon[49373]: osdmap e87: 8 total, 7 up, 8 in 2024-09-17T12:03:38.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:37 smithi165 ceph-mon[49373]: from='osd.1 [v2:172.21.15.45:6810/1032158886,v1:172.21.15.45:6811/1032158886]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi045", "root=default"]} : dispatch 2024-09-17T12:03:38.751 INFO:teuthology.orchestra.run.smithi045.stdout: 5 173401 9.66 MB/sec execute 339 sec latency 806.576 ms 2024-09-17T12:03:39.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:38 smithi165 ceph-mon[49373]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T12:03:39.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:38 smithi165 ceph-mon[49373]: osd.1 [v2:172.21.15.45:6810/1032158886,v1:172.21.15.45:6811/1032158886] boot 2024-09-17T12:03:39.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:38 smithi165 ceph-mon[49373]: osdmap e88: 8 total, 8 up, 8 in 2024-09-17T12:03:39.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:38 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-17T12:03:39.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:38 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 184/1632 objects degraded (11.275%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T12:03:39.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:38 smithi165 ceph-mon[49373]: osdmap e89: 8 total, 8 up, 8 in 2024-09-17T12:03:39.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:38 smithi045 ceph-mon[63880]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T12:03:39.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:38 smithi045 ceph-mon[63880]: osd.1 [v2:172.21.15.45:6810/1032158886,v1:172.21.15.45:6811/1032158886] boot 2024-09-17T12:03:39.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:38 smithi045 ceph-mon[63880]: osdmap e88: 8 total, 8 up, 8 in 2024-09-17T12:03:39.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:38 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-17T12:03:39.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:38 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 184/1632 objects degraded (11.275%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T12:03:39.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:38 smithi045 ceph-mon[63880]: osdmap e89: 8 total, 8 up, 8 in 2024-09-17T12:03:39.421 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T12:03:39.751 INFO:teuthology.orchestra.run.smithi045.stdout: 5 173521 9.64 MB/sec execute 340 sec latency 1461.580 ms 2024-09-17T12:03:40.148 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:39 smithi045 ceph-mon[63880]: pgmap v104: 97 pgs: 2 activating, 2 remapped+peering, 13 active+undersized, 22 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 552 op/s; 158/1614 objects degraded (9.789%) 2024-09-17T12:03:40.148 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:39 smithi045 ceph-mon[63880]: osdmap e90: 8 total, 8 up, 8 in 2024-09-17T12:03:40.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:39 smithi165 ceph-mon[49373]: pgmap v104: 97 pgs: 2 activating, 2 remapped+peering, 13 active+undersized, 22 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.2 MiB/s wr, 552 op/s; 158/1614 objects degraded (9.789%) 2024-09-17T12:03:40.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:39 smithi165 ceph-mon[49373]: osdmap e90: 8 total, 8 up, 8 in 2024-09-17T12:03:40.497 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T12:03:40.497 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (5m) 5s ago 14m 20.3M - 0.27.0 11f11916f8cd 2a2a54402d2a 2024-09-17T12:03:40.497 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (102s) 5s ago 14m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 e545cbf7736d 2024-09-17T12:03:40.497 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (96s) 91s ago 13m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 c1e89e114e4d 2024-09-17T12:03:40.497 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (4m) 5s ago 14m 58.9M - dad864ee21e9 371aaf55379f 2024-09-17T12:03:40.497 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (8m) 5s ago 8m 37.8M - 17.2.0 e1d6a67b021e 6a2a4a998dd1 2024-09-17T12:03:40.497 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (8m) 91s ago 8m 16.0M - 17.2.0 e1d6a67b021e d4c2987b9bac 2024-09-17T12:03:40.497 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:8443,9283,8765 running (4m) 5s ago 15m 517M - 19.3.0-4937-g74984932 6c1fcab85602 ae2f507bd67d 2024-09-17T12:03:40.497 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283,8765 running (3m) 91s ago 13m 453M - 19.3.0-4937-g74984932 6c1fcab85602 53403866652e 2024-09-17T12:03:40.498 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (2m) 5s ago 15m 49.9M 2048M 19.3.0-4937-g74984932 6c1fcab85602 737eec78e0f0 2024-09-17T12:03:40.498 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (2m) 91s ago 13m 36.1M 2048M 19.3.0-4937-g74984932 6c1fcab85602 249eedb4123e 2024-09-17T12:03:40.498 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (7m) 5s ago 7m 277M - 4.0 e1d6a67b021e b8fc749ebb71 2024-09-17T12:03:40.498 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (7m) 91s ago 7m 83.8M - 4.0 e1d6a67b021e 771b27764432 2024-09-17T12:03:40.498 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (4m) 5s ago 13m 11.2M - 1.7.0 72c9c2088986 989b5738cb69 2024-09-17T12:03:40.498 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (4m) 91s ago 13m 11.9M - 1.7.0 72c9c2088986 e1e8ffab3f21 2024-09-17T12:03:40.498 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (63s) 5s ago 12m 218M 1937M 19.3.0-4937-g74984932 6c1fcab85602 20dafd01bd12 2024-09-17T12:03:40.498 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (14s) 5s ago 12m 14.3M 1937M 19.3.0-4937-g74984932 6c1fcab85602 d39be5014418 2024-09-17T12:03:40.498 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (11m) 5s ago 11m 403M 1937M 17.2.0 e1d6a67b021e 21fe0d9e5ced 2024-09-17T12:03:40.498 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (11m) 5s ago 11m 538M 1937M 17.2.0 e1d6a67b021e 92a2389488e4 2024-09-17T12:03:40.498 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (10m) 91s ago 10m 494M 2704M 17.2.0 e1d6a67b021e e29eb74df25a 2024-09-17T12:03:40.498 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (10m) 91s ago 10m 544M 2704M 17.2.0 e1d6a67b021e 5d900eb9165b 2024-09-17T12:03:40.499 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (10m) 91s ago 10m 466M 2704M 17.2.0 e1d6a67b021e fae3e1bae466 2024-09-17T12:03:40.499 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (9m) 91s ago 9m 371M 2704M 17.2.0 e1d6a67b021e 71b9f2cdb810 2024-09-17T12:03:40.499 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (3m) 5s ago 13m 55.9M - 2.51.0 1d3b7f56885b 61b064ad6609 2024-09-17T12:03:40.751 INFO:teuthology.orchestra.run.smithi045.stdout: 5 173708 9.64 MB/sec execute 341 sec latency 551.694 ms 2024-09-17T12:03:40.941 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:03:40.941 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T12:03:40.941 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:03:40.941 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:03:40.941 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T12:03:40.942 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:03:40.942 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:03:40.942 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T12:03:40.942 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-17T12:03:40.942 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:03:40.942 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:03:40.942 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T12:03:40.942 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T12:03:40.942 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:03:40.942 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T12:03:40.942 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-17T12:03:40.942 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 6 2024-09-17T12:03:40.942 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T12:03:40.942 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:03:41.233 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:40 smithi045 ceph-mon[63880]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:03:41.233 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:40 smithi045 ceph-mon[63880]: from='client.44189 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:03:41.233 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:40 smithi045 ceph-mon[63880]: osdmap e91: 8 total, 8 up, 8 in 2024-09-17T12:03:41.233 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:40 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/2924283888' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:03:41.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:40 smithi165 ceph-mon[49373]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:03:41.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:40 smithi165 ceph-mon[49373]: from='client.44189 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:03:41.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:40 smithi165 ceph-mon[49373]: osdmap e91: 8 total, 8 up, 8 in 2024-09-17T12:03:41.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:40 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/2924283888' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:03:41.399 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:03:41.399 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T12:03:41.399 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T12:03:41.399 INFO:teuthology.orchestra.run.smithi045.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T12:03:41.400 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [ 2024-09-17T12:03:41.400 INFO:teuthology.orchestra.run.smithi045.stdout: "crash", 2024-09-17T12:03:41.400 INFO:teuthology.orchestra.run.smithi045.stdout: "mon", 2024-09-17T12:03:41.400 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr" 2024-09-17T12:03:41.400 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-17T12:03:41.400 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "8/23 daemons upgraded", 2024-09-17T12:03:41.400 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T12:03:41.400 INFO:teuthology.orchestra.run.smithi045.stdout: "is_paused": false 2024-09-17T12:03:41.400 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:03:41.751 INFO:teuthology.orchestra.run.smithi045.stdout: 5 173900 9.63 MB/sec execute 342 sec latency 1032.516 ms 2024-09-17T12:03:41.988 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:41 smithi045 ceph-mon[63880]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:03:41.988 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:41 smithi045 ceph-mon[63880]: pgmap v107: 97 pgs: 2 activating, 2 remapped+peering, 13 active+undersized, 22 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 1.6 MiB/s wr, 201 op/s; 158/1614 objects degraded (9.789%) 2024-09-17T12:03:41.988 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:41 smithi045 ceph-mon[63880]: osdmap e92: 8 total, 8 up, 8 in 2024-09-17T12:03:41.988 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:41 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:41.989 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:41 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:41.989 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:41 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:03:41.989 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:41 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:03:41.989 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:41 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:41.989 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:41 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:42.145 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_WARN Degraded data redundancy: 158/1614 objects degraded (9.789%), 22 pgs degraded 2024-09-17T12:03:42.145 INFO:teuthology.orchestra.run.smithi045.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 158/1614 objects degraded (9.789%), 22 pgs degraded 2024-09-17T12:03:42.145 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-09-17T12:03:42.145 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-09-17T12:03:42.145 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-09-17T12:03:42.145 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.8 is active+undersized+degraded, acting [7,0] 2024-09-17T12:03:42.146 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-09-17T12:03:42.146 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.a is active+undersized+degraded, acting [3,7] 2024-09-17T12:03:42.146 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.d is active+undersized+degraded, acting [4,3] 2024-09-17T12:03:42.146 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.10 is active+undersized+degraded, acting [2,0] 2024-09-17T12:03:42.146 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.11 is active+undersized+degraded, acting [6,4] 2024-09-17T12:03:42.146 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-09-17T12:03:42.146 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-09-17T12:03:42.146 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-09-17T12:03:42.146 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-09-17T12:03:42.146 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.a is active+undersized+degraded, acting [6,4] 2024-09-17T12:03:42.146 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.e is active+undersized+degraded, acting [7,4] 2024-09-17T12:03:42.146 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-09-17T12:03:42.146 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-09-17T12:03:42.147 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-09-17T12:03:42.147 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.1a is active+undersized+degraded, acting [4,2] 2024-09-17T12:03:42.147 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-09-17T12:03:42.147 INFO:teuthology.orchestra.run.smithi045.stdout: pg 4.a is active+undersized+degraded, acting [6,7] 2024-09-17T12:03:42.147 INFO:teuthology.orchestra.run.smithi045.stdout: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-09-17T12:03:42.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:41 smithi165 ceph-mon[49373]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:03:42.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:41 smithi165 ceph-mon[49373]: pgmap v107: 97 pgs: 2 activating, 2 remapped+peering, 13 active+undersized, 22 active+undersized+degraded, 58 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 1.6 MiB/s wr, 201 op/s; 158/1614 objects degraded (9.789%) 2024-09-17T12:03:42.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:41 smithi165 ceph-mon[49373]: osdmap e92: 8 total, 8 up, 8 in 2024-09-17T12:03:42.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:41 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:42.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:41 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:42.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:41 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:03:42.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:41 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:03:42.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:41 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:42.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:41 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:42.751 INFO:teuthology.orchestra.run.smithi045.stdout: 5 174371 9.65 MB/sec execute 343 sec latency 299.082 ms 2024-09-17T12:03:43.003 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:42 smithi045 ceph-mon[63880]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:03:43.003 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:42 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/1268640140' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:03:43.003 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:42 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:03:43.003 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:42 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:03:43.003 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:42 smithi045 ceph-mon[63880]: osdmap e93: 8 total, 8 up, 8 in 2024-09-17T12:03:43.003 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:42 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:03:43.003 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:42 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:03:43.004 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:42 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-17T12:03:43.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:42 smithi165 ceph-mon[49373]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:03:43.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:42 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/1268640140' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:03:43.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:42 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:03:43.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:42 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:03:43.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:42 smithi165 ceph-mon[49373]: osdmap e93: 8 total, 8 up, 8 in 2024-09-17T12:03:43.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:42 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:03:43.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:42 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:03:43.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:42 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-17T12:03:43.751 INFO:teuthology.orchestra.run.smithi045.stdout: 5 174757 9.64 MB/sec execute 344 sec latency 786.919 ms 2024-09-17T12:03:44.007 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:43 smithi045 ceph-mon[63880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T12:03:44.007 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:43 smithi045 ceph-mon[63880]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-17T12:03:44.007 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:43 smithi045 ceph-mon[63880]: pgmap v110: 97 pgs: 1 active+undersized+remapped, 2 peering, 2 activating, 1 remapped+peering, 91 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 324 op/s; 3.7 MiB/s, 75 keys/s, 63 objects/s recovering 2024-09-17T12:03:44.007 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:43 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 158/1614 objects degraded (9.789%), 22 pgs degraded (PG_DEGRADED) 2024-09-17T12:03:44.007 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:43 smithi045 ceph-mon[63880]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 158/1614 objects degraded (9.789%), 22 pgs degraded) 2024-09-17T12:03:44.008 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:43 smithi045 ceph-mon[63880]: Cluster is now healthy 2024-09-17T12:03:44.008 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:43 smithi045 ceph-mon[63880]: osdmap e94: 8 total, 8 up, 8 in 2024-09-17T12:03:44.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:43 smithi165 ceph-mon[49373]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T12:03:44.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:43 smithi165 ceph-mon[49373]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-17T12:03:44.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:43 smithi165 ceph-mon[49373]: pgmap v110: 97 pgs: 1 active+undersized+remapped, 2 peering, 2 activating, 1 remapped+peering, 91 active+clean; 348 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 324 op/s; 3.7 MiB/s, 75 keys/s, 63 objects/s recovering 2024-09-17T12:03:44.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:43 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 158/1614 objects degraded (9.789%), 22 pgs degraded (PG_DEGRADED) 2024-09-17T12:03:44.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:43 smithi165 ceph-mon[49373]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 158/1614 objects degraded (9.789%), 22 pgs degraded) 2024-09-17T12:03:44.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:43 smithi165 ceph-mon[49373]: Cluster is now healthy 2024-09-17T12:03:44.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:43 smithi165 ceph-mon[49373]: osdmap e94: 8 total, 8 up, 8 in 2024-09-17T12:03:44.751 INFO:teuthology.orchestra.run.smithi045.stdout: 5 175201 9.63 MB/sec execute 345 sec latency 43.243 ms 2024-09-17T12:03:45.751 INFO:teuthology.orchestra.run.smithi045.stdout: 5 175546 9.62 MB/sec execute 346 sec latency 71.990 ms 2024-09-17T12:03:46.036 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:45 smithi045 ceph-mon[63880]: pgmap v112: 97 pgs: 1 active+undersized+remapped, 2 peering, 94 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.9 MiB/s wr, 296 op/s; 3.3 MiB/s, 64 keys/s, 59 objects/s recovering 2024-09-17T12:03:46.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:45 smithi165 ceph-mon[49373]: pgmap v112: 97 pgs: 1 active+undersized+remapped, 2 peering, 94 active+clean; 349 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.9 MiB/s wr, 296 op/s; 3.3 MiB/s, 64 keys/s, 59 objects/s recovering 2024-09-17T12:03:46.751 INFO:teuthology.orchestra.run.smithi045.stdout: 5 175982 9.64 MB/sec execute 347 sec latency 47.657 ms 2024-09-17T12:03:47.751 INFO:teuthology.orchestra.run.smithi045.stdout: 5 176414 9.64 MB/sec execute 348 sec latency 71.015 ms 2024-09-17T12:03:48.130 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:47 smithi045 ceph-mon[63880]: pgmap v113: 97 pgs: 1 active+undersized+remapped, 2 peering, 94 active+clean; 352 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.6 MiB/s wr, 527 op/s; 2.5 MiB/s, 50 keys/s, 46 objects/s recovering 2024-09-17T12:03:48.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:47 smithi165 ceph-mon[49373]: pgmap v113: 97 pgs: 1 active+undersized+remapped, 2 peering, 94 active+clean; 352 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.6 MiB/s wr, 527 op/s; 2.5 MiB/s, 50 keys/s, 46 objects/s recovering 2024-09-17T12:03:48.751 INFO:teuthology.orchestra.run.smithi045.stdout: 5 176931 9.66 MB/sec execute 349 sec latency 39.404 ms 2024-09-17T12:03:49.752 INFO:teuthology.orchestra.run.smithi045.stdout: 5 177502 9.68 MB/sec execute 350 sec latency 26.470 ms 2024-09-17T12:03:50.145 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:49 smithi045 ceph-mon[63880]: pgmap v114: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.2 MiB/s wr, 585 op/s; 2.0 MiB/s, 39 keys/s, 36 objects/s recovering 2024-09-17T12:03:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:49 smithi165 ceph-mon[49373]: pgmap v114: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.2 MiB/s wr, 585 op/s; 2.0 MiB/s, 39 keys/s, 36 objects/s recovering 2024-09-17T12:03:50.752 INFO:teuthology.orchestra.run.smithi045.stdout: 5 178067 9.71 MB/sec execute 351 sec latency 48.512 ms 2024-09-17T12:03:51.752 INFO:teuthology.orchestra.run.smithi045.stdout: 5 178714 9.71 MB/sec execute 352 sec latency 50.099 ms 2024-09-17T12:03:52.209 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:51 smithi045 ceph-mon[63880]: pgmap v115: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.7 MiB/s wr, 367 op/s; 56 KiB/s, 2 objects/s recovering 2024-09-17T12:03:52.209 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:52.209 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:03:52.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:51 smithi165 ceph-mon[49373]: pgmap v115: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.7 MiB/s wr, 367 op/s; 56 KiB/s, 2 objects/s recovering 2024-09-17T12:03:52.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:51 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:52.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:51 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:03:52.752 INFO:teuthology.orchestra.run.smithi045.stdout: 5 179079 9.70 MB/sec execute 353 sec latency 41.024 ms 2024-09-17T12:03:53.752 INFO:teuthology.orchestra.run.smithi045.stdout: 5 179408 9.70 MB/sec execute 354 sec latency 38.155 ms 2024-09-17T12:03:53.998 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:53 smithi045 ceph-mon[63880]: pgmap v116: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.6 MiB/s wr, 680 op/s; 49 KiB/s, 2 objects/s recovering 2024-09-17T12:03:54.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:53 smithi165 ceph-mon[49373]: pgmap v116: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.6 MiB/s wr, 680 op/s; 49 KiB/s, 2 objects/s recovering 2024-09-17T12:03:54.752 INFO:teuthology.orchestra.run.smithi045.stdout: 5 179746 9.70 MB/sec execute 355 sec latency 87.115 ms 2024-09-17T12:03:55.752 INFO:teuthology.orchestra.run.smithi045.stdout: 5 180176 9.72 MB/sec execute 356 sec latency 50.437 ms 2024-09-17T12:03:56.012 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:55 smithi045 ceph-mon[63880]: pgmap v117: 97 pgs: 97 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.9 MiB/s wr, 595 op/s 2024-09-17T12:03:56.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:55 smithi165 ceph-mon[49373]: pgmap v117: 97 pgs: 97 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.9 MiB/s wr, 595 op/s 2024-09-17T12:03:56.752 INFO:teuthology.orchestra.run.smithi045.stdout: 5 180665 9.74 MB/sec execute 357 sec latency 58.311 ms 2024-09-17T12:03:57.587 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:57 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:57.587 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:57 smithi045 ceph-mon[63880]: pgmap v118: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.9 MiB/s wr, 589 op/s 2024-09-17T12:03:57.587 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:57 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-17T12:03:57.752 INFO:teuthology.orchestra.run.smithi045.stdout: 5 180966 9.73 MB/sec execute 358 sec latency 51.363 ms 2024-09-17T12:03:57.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:57 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:03:57.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:57 smithi165 ceph-mon[49373]: pgmap v118: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.9 MiB/s wr, 589 op/s 2024-09-17T12:03:57.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:57 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-17T12:03:58.602 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:58 smithi045 ceph-mon[63880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T12:03:58.602 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:58 smithi045 ceph-mon[63880]: Upgrade: osd.2 is safe to restart 2024-09-17T12:03:58.752 INFO:teuthology.orchestra.run.smithi045.stdout: 5 181365 9.73 MB/sec execute 359 sec latency 59.412 ms 2024-09-17T12:03:58.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:58 smithi165 ceph-mon[49373]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T12:03:58.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:58 smithi165 ceph-mon[49373]: Upgrade: osd.2 is safe to restart 2024-09-17T12:03:59.615 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:03:59 smithi045 ceph-mon[63880]: pgmap v119: 97 pgs: 97 active+clean; 352 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 7.4 MiB/s wr, 658 op/s 2024-09-17T12:03:59.752 INFO:teuthology.orchestra.run.smithi045.stdout: 5 181654 9.73 MB/sec execute 360 sec latency 57.702 ms 2024-09-17T12:03:59.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:03:59 smithi165 ceph-mon[49373]: pgmap v119: 97 pgs: 97 active+clean; 352 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 7.4 MiB/s wr, 658 op/s 2024-09-17T12:04:00.663 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:00 smithi045 ceph-mon[63880]: Upgrade: Updating osd.2 2024-09-17T12:04:00.663 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:00 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:00.663 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:00 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-17T12:04:00.663 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:00 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:04:00.663 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:00 smithi045 ceph-mon[63880]: Deploying daemon osd.2 on smithi045 2024-09-17T12:04:00.752 INFO:teuthology.orchestra.run.smithi045.stdout: 5 181898 9.71 MB/sec execute 361 sec latency 83.852 ms 2024-09-17T12:04:01.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:00 smithi165 ceph-mon[49373]: Upgrade: Updating osd.2 2024-09-17T12:04:01.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:00 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:01.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:00 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-17T12:04:01.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:00 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:04:01.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:00 smithi165 ceph-mon[49373]: Deploying daemon osd.2 on smithi045 2024-09-17T12:04:01.680 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:01 smithi045 ceph-mon[63880]: pgmap v120: 97 pgs: 97 active+clean; 352 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 547 op/s 2024-09-17T12:04:01.752 INFO:teuthology.orchestra.run.smithi045.stdout: 5 182329 9.71 MB/sec execute 362 sec latency 58.891 ms 2024-09-17T12:04:02.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:01 smithi165 ceph-mon[49373]: pgmap v120: 97 pgs: 97 active+clean; 352 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 547 op/s 2024-09-17T12:04:02.753 INFO:teuthology.orchestra.run.smithi045.stdout: 5 183072 9.73 MB/sec execute 363 sec latency 26.599 ms 2024-09-17T12:04:03.753 INFO:teuthology.orchestra.run.smithi045.stdout: 5 183765 9.77 MB/sec execute 364 sec latency 26.336 ms 2024-09-17T12:04:03.965 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:03 smithi045 ceph-mon[63880]: pgmap v121: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 758 op/s 2024-09-17T12:04:04.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:03 smithi165 ceph-mon[49373]: pgmap v121: 97 pgs: 97 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 758 op/s 2024-09-17T12:04:04.753 INFO:teuthology.orchestra.run.smithi045.stdout: 5 184315 9.79 MB/sec execute 365 sec latency 40.254 ms 2024-09-17T12:04:05.493 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:05 smithi045 systemd[1]: Stopping Ceph osd.2 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:04:05.745 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:05 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2[40781]: 2024-09-17T12:04:05.635+0000 7f151164f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T12:04:05.745 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:05 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2[40781]: 2024-09-17T12:04:05.635+0000 7f151164f700 -1 osd.2 94 *** Got signal Terminated *** 2024-09-17T12:04:05.745 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:05 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2[40781]: 2024-09-17T12:04:05.635+0000 7f151164f700 -1 osd.2 94 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T12:04:05.753 INFO:teuthology.orchestra.run.smithi045.stdout: 5 184538 9.78 MB/sec execute 366 sec latency 112.815 ms 2024-09-17T12:04:06.005 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:05 smithi045 ceph-mon[63880]: pgmap v122: 97 pgs: 97 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 474 op/s 2024-09-17T12:04:06.005 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:05 smithi045 ceph-mon[63880]: osd.2 marked itself down and dead 2024-09-17T12:04:06.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:05 smithi165 ceph-mon[49373]: pgmap v122: 97 pgs: 97 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 474 op/s 2024-09-17T12:04:06.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:05 smithi165 ceph-mon[49373]: osd.2 marked itself down and dead 2024-09-17T12:04:06.753 INFO:teuthology.orchestra.run.smithi045.stdout: 5 184824 9.77 MB/sec execute 367 sec latency 722.843 ms 2024-09-17T12:04:06.781 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:06 smithi045 podman[73044]: 2024-09-17 12:04:06.555496499 +0000 UTC m=+1.056589440 container died 21fe0d9e5ced9cd98f6135a4a5305bab69392082a07dc5ab6d9ea9bfd129ad1f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, vcs-type=git, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, name=centos-stream, release=754, maintainer=Guillaume Abrioux , version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container) 2024-09-17T12:04:07.040 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:06 smithi045 ceph-mon[63880]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T12:04:07.040 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:06 smithi045 ceph-mon[63880]: osdmap e95: 8 total, 7 up, 8 in 2024-09-17T12:04:07.040 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:06 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:07.040 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:06 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:04:07.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:06 smithi165 ceph-mon[49373]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T12:04:07.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:06 smithi165 ceph-mon[49373]: osdmap e95: 8 total, 7 up, 8 in 2024-09-17T12:04:07.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:06 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:07.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:06 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:04:07.753 INFO:teuthology.orchestra.run.smithi045.stdout: 5 185148 9.78 MB/sec execute 368 sec latency 964.591 ms 2024-09-17T12:04:07.793 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:07 smithi045 podman[73044]: 2024-09-17 12:04:07.651945041 +0000 UTC m=+2.153037961 container cleanup 21fe0d9e5ced9cd98f6135a4a5305bab69392082a07dc5ab6d9ea9bfd129ad1f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, release=754, maintainer=Guillaume Abrioux , architecture=x86_64, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, build-date=2022-05-03T08:36:31.336870) 2024-09-17T12:04:07.793 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:07 smithi045 bash[73044]: ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2 2024-09-17T12:04:08.052 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:07 smithi045 ceph-mon[63880]: pgmap v124: 97 pgs: 3 peering, 6 stale+active+clean, 88 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 5.7 MiB/s wr, 547 op/s 2024-09-17T12:04:08.052 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:07 smithi045 ceph-mon[63880]: osdmap e96: 8 total, 7 up, 8 in 2024-09-17T12:04:08.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:07 smithi165 ceph-mon[49373]: pgmap v124: 97 pgs: 3 peering, 6 stale+active+clean, 88 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 5.7 MiB/s wr, 547 op/s 2024-09-17T12:04:08.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:07 smithi165 ceph-mon[49373]: osdmap e96: 8 total, 7 up, 8 in 2024-09-17T12:04:08.309 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:08 smithi045 podman[73059]: 2024-09-17 12:04:08.116054789 +0000 UTC m=+1.558101901 container remove 21fe0d9e5ced9cd98f6135a4a5305bab69392082a07dc5ab6d9ea9bfd129ad1f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2, release=754, io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, com.redhat.component=centos-stream-container, architecture=x86_64, distribution-scope=public, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, vendor=Red Hat, Inc., ceph=True, io.openshift.expose-services=, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-17T12:04:08.753 INFO:teuthology.orchestra.run.smithi045.stdout: 5 185435 9.76 MB/sec execute 369 sec latency 73.101 ms 2024-09-17T12:04:08.815 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:08 smithi045 podman[73138]: 2024-09-17 12:04:08.622678195 +0000 UTC m=+0.023617215 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:04:08.815 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:08 smithi045 podman[73138]: 2024-09-17 12:04:08.758797311 +0000 UTC m=+0.159736315 container create f2a8fc892c3ef4d6b271fc941b57bcfebe936bb5ff61a8e2bee2c1e2aef958dd (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493) 2024-09-17T12:04:09.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:09 smithi045 ceph-mon[63880]: pgmap v126: 97 pgs: 2 active+undersized, 1 activating+undersized, 18 peering, 4 active+undersized+degraded, 72 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.9 MiB/s wr, 752 op/s; 52/1716 objects degraded (3.030%) 2024-09-17T12:04:09.340 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:09 smithi045 podman[73138]: 2024-09-17 12:04:09.129095812 +0000 UTC m=+0.530034835 container init f2a8fc892c3ef4d6b271fc941b57bcfebe936bb5ff61a8e2bee2c1e2aef958dd (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=main-7498493, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T12:04:09.341 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:09 smithi045 podman[73138]: 2024-09-17 12:04:09.13895979 +0000 UTC m=+0.539898802 container start f2a8fc892c3ef4d6b271fc941b57bcfebe936bb5ff61a8e2bee2c1e2aef958dd (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-17T12:04:09.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:09 smithi165 ceph-mon[49373]: pgmap v126: 97 pgs: 2 active+undersized, 1 activating+undersized, 18 peering, 4 active+undersized+degraded, 72 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.9 MiB/s wr, 752 op/s; 52/1716 objects degraded (3.030%) 2024-09-17T12:04:09.589 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:09 smithi045 podman[73138]: 2024-09-17 12:04:09.40471442 +0000 UTC m=+0.805653435 container attach f2a8fc892c3ef4d6b271fc941b57bcfebe936bb5ff61a8e2bee2c1e2aef958dd (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, RELEASE=main-7498493, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T12:04:09.590 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:09 smithi045 podman[73138]: 2024-09-17 12:04:09.463879434 +0000 UTC m=+0.864818438 container died f2a8fc892c3ef4d6b271fc941b57bcfebe936bb5ff61a8e2bee2c1e2aef958dd (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-17T12:04:09.753 INFO:teuthology.orchestra.run.smithi045.stdout: 5 185698 9.76 MB/sec execute 370 sec latency 49.517 ms 2024-09-17T12:04:10.346 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:10 smithi045 ceph-mon[63880]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-17T12:04:10.346 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:10 smithi045 ceph-mon[63880]: Health check failed: Degraded data redundancy: 52/1716 objects degraded (3.030%), 4 pgs degraded (PG_DEGRADED) 2024-09-17T12:04:10.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:10 smithi165 ceph-mon[49373]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-17T12:04:10.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:10 smithi165 ceph-mon[49373]: Health check failed: Degraded data redundancy: 52/1716 objects degraded (3.030%), 4 pgs degraded (PG_DEGRADED) 2024-09-17T12:04:10.612 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:10 smithi045 podman[73160]: 2024-09-17 12:04:10.537065717 +0000 UTC m=+1.065606482 container remove f2a8fc892c3ef4d6b271fc941b57bcfebe936bb5ff61a8e2bee2c1e2aef958dd (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-17T12:04:10.612 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:10 smithi045 systemd[1]: ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-17T12:04:10.612 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:10 smithi045 systemd[1]: Stopped Ceph osd.2 for a12537de-74ea-11ef-bceb-c7b262605968. 2024-09-17T12:04:10.612 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:10 smithi045 systemd[1]: ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.2.service: Consumed 2min 19.472s CPU time. 2024-09-17T12:04:10.753 INFO:teuthology.orchestra.run.smithi045.stdout: 5 185946 9.73 MB/sec execute 371 sec latency 74.866 ms 2024-09-17T12:04:11.127 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:10 smithi045 systemd[1]: Starting Ceph osd.2 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:04:11.380 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:11 smithi045 ceph-mon[63880]: pgmap v127: 97 pgs: 2 active+undersized, 1 activating+undersized, 18 peering, 4 active+undersized+degraded, 72 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 5.1 MiB/s wr, 437 op/s; 52/1716 objects degraded (3.030%) 2024-09-17T12:04:11.380 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:11 smithi045 podman[73251]: 2024-09-17 12:04:11.029849294 +0000 UTC m=+0.019291219 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:04:11.380 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:11 smithi045 podman[73251]: 2024-09-17 12:04:11.180274663 +0000 UTC m=+0.169716581 container create d65d09a4979be7042a3ccc75f0c170511c232f016cd9e4d04de29d38161c7ca4 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=main-7498493, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-17T12:04:11.631 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:11 smithi045 podman[73251]: 2024-09-17 12:04:11.533874644 +0000 UTC m=+0.523316558 container init d65d09a4979be7042a3ccc75f0c170511c232f016cd9e4d04de29d38161c7ca4 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=main-7498493, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T12:04:11.631 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:11 smithi045 podman[73251]: 2024-09-17 12:04:11.542826861 +0000 UTC m=+0.532268781 container start d65d09a4979be7042a3ccc75f0c170511c232f016cd9e4d04de29d38161c7ca4 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, RELEASE=main-7498493, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-17T12:04:11.753 INFO:teuthology.orchestra.run.smithi045.stdout: 5 186220 9.72 MB/sec execute 372 sec latency 49.017 ms 2024-09-17T12:04:11.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:11 smithi165 ceph-mon[49373]: pgmap v127: 97 pgs: 2 active+undersized, 1 activating+undersized, 18 peering, 4 active+undersized+degraded, 72 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 5.1 MiB/s wr, 437 op/s; 52/1716 objects degraded (3.030%) 2024-09-17T12:04:11.896 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:11 smithi045 podman[73251]: 2024-09-17 12:04:11.633855721 +0000 UTC m=+0.623297645 container attach d65d09a4979be7042a3ccc75f0c170511c232f016cd9e4d04de29d38161c7ca4 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-activate, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-17T12:04:11.896 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:11 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-activate[73265]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:04:11.897 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:11 smithi045 bash[73251]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:04:11.897 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:11 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-activate[73265]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:04:11.897 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:11 smithi045 bash[73251]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:04:12.645 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T12:04:12.680 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:12 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-activate[73265]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T12:04:12.680 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:12 smithi045 bash[73251]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T12:04:12.680 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:12 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-activate[73265]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T12:04:12.680 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:12 smithi045 bash[73251]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T12:04:12.753 INFO:teuthology.orchestra.run.smithi045.stdout: 5 186452 9.71 MB/sec execute 373 sec latency 53.702 ms 2024-09-17T12:04:13.188 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:13 smithi045 ceph-mon[63880]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-17T12:04:13.188 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:12 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-activate[73265]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T12:04:13.188 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:12 smithi045 bash[73251]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T12:04:13.188 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:12 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-activate[73265]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T12:04:13.188 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:12 smithi045 bash[73251]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T12:04:13.189 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:12 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-activate[73265]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-17T12:04:13.189 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:12 smithi045 bash[73251]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-17T12:04:13.189 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:12 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-activate[73265]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T12:04:13.189 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:12 smithi045 bash[73251]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T12:04:13.189 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:13 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-activate[73265]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-17T12:04:13.189 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:13 smithi045 bash[73251]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-17T12:04:13.189 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:13 smithi045 podman[73251]: 2024-09-17 12:04:13.038300257 +0000 UTC m=+2.027742176 container died d65d09a4979be7042a3ccc75f0c170511c232f016cd9e4d04de29d38161c7ca4 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, RELEASE=main-7498493, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-17T12:04:13.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:13 smithi165 ceph-mon[49373]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-17T12:04:13.677 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T12:04:13.677 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (5m) 38s ago 14m 20.3M - 0.27.0 11f11916f8cd 2a2a54402d2a 2024-09-17T12:04:13.677 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (2m) 38s ago 14m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 e545cbf7736d 2024-09-17T12:04:13.677 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (2m) 2m ago 14m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 c1e89e114e4d 2024-09-17T12:04:13.677 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (5m) 38s ago 14m 58.9M - dad864ee21e9 371aaf55379f 2024-09-17T12:04:13.677 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (8m) 38s ago 8m 37.8M - 17.2.0 e1d6a67b021e 6a2a4a998dd1 2024-09-17T12:04:13.677 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (8m) 2m ago 8m 16.0M - 17.2.0 e1d6a67b021e d4c2987b9bac 2024-09-17T12:04:13.677 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:8443,9283,8765 running (4m) 38s ago 16m 517M - 19.3.0-4937-g74984932 6c1fcab85602 ae2f507bd67d 2024-09-17T12:04:13.678 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283,8765 running (3m) 2m ago 14m 453M - 19.3.0-4937-g74984932 6c1fcab85602 53403866652e 2024-09-17T12:04:13.678 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (3m) 38s ago 16m 49.9M 2048M 19.3.0-4937-g74984932 6c1fcab85602 737eec78e0f0 2024-09-17T12:04:13.678 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (3m) 2m ago 14m 36.1M 2048M 19.3.0-4937-g74984932 6c1fcab85602 249eedb4123e 2024-09-17T12:04:13.678 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (8m) 38s ago 8m 277M - 4.0 e1d6a67b021e b8fc749ebb71 2024-09-17T12:04:13.678 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (8m) 2m ago 8m 83.8M - 4.0 e1d6a67b021e 771b27764432 2024-09-17T12:04:13.678 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (5m) 38s ago 14m 11.2M - 1.7.0 72c9c2088986 989b5738cb69 2024-09-17T12:04:13.678 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (4m) 2m ago 13m 11.9M - 1.7.0 72c9c2088986 e1e8ffab3f21 2024-09-17T12:04:13.678 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (97s) 38s ago 13m 218M 1937M 19.3.0-4937-g74984932 6c1fcab85602 20dafd01bd12 2024-09-17T12:04:13.678 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (48s) 38s ago 12m 14.3M 1937M 19.3.0-4937-g74984932 6c1fcab85602 d39be5014418 2024-09-17T12:04:13.678 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (12m) 38s ago 12m 403M 1937M 17.2.0 e1d6a67b021e 21fe0d9e5ced 2024-09-17T12:04:13.678 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (11m) 38s ago 11m 538M 1937M 17.2.0 e1d6a67b021e 92a2389488e4 2024-09-17T12:04:13.678 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (11m) 2m ago 11m 494M 2704M 17.2.0 e1d6a67b021e e29eb74df25a 2024-09-17T12:04:13.679 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (11m) 2m ago 11m 544M 2704M 17.2.0 e1d6a67b021e 5d900eb9165b 2024-09-17T12:04:13.679 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (10m) 2m ago 10m 466M 2704M 17.2.0 e1d6a67b021e fae3e1bae466 2024-09-17T12:04:13.679 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (10m) 2m ago 10m 371M 2704M 17.2.0 e1d6a67b021e 71b9f2cdb810 2024-09-17T12:04:13.679 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (3m) 38s ago 13m 55.9M - 2.51.0 1d3b7f56885b 61b064ad6609 2024-09-17T12:04:13.754 INFO:teuthology.orchestra.run.smithi045.stdout: 5 186746 9.72 MB/sec execute 374 sec latency 51.656 ms 2024-09-17T12:04:14.224 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:14 smithi045 ceph-mon[63880]: from='client.34280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:04:14.224 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:14 smithi045 ceph-mon[63880]: pgmap v128: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.4 MiB/s wr, 669 op/s; 141/1581 objects degraded (8.918%) 2024-09-17T12:04:14.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:14 smithi165 ceph-mon[49373]: from='client.34280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:04:14.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:14 smithi165 ceph-mon[49373]: pgmap v128: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.4 MiB/s wr, 669 op/s; 141/1581 objects degraded (8.918%) 2024-09-17T12:04:14.436 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:04:14.436 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T12:04:14.437 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:04:14.437 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:04:14.437 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T12:04:14.437 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:04:14.437 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:04:14.437 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T12:04:14.437 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-17T12:04:14.437 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:04:14.437 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:04:14.438 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T12:04:14.438 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T12:04:14.438 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:04:14.438 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T12:04:14.438 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-17T12:04:14.438 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 6 2024-09-17T12:04:14.438 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T12:04:14.438 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:04:14.479 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:14 smithi045 podman[73445]: 2024-09-17 12:04:14.234122985 +0000 UTC m=+1.188440783 container remove d65d09a4979be7042a3ccc75f0c170511c232f016cd9e4d04de29d38161c7ca4 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=main-7498493, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-17T12:04:14.731 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:14 smithi045 podman[73546]: 2024-09-17 12:04:14.411113809 +0000 UTC m=+0.024911946 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:04:14.731 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:14 smithi045 podman[73546]: 2024-09-17 12:04:14.585985223 +0000 UTC m=+0.199783358 container create e13c2ecd369ac53b2f9288427f2a071d5c802d3effe0cbc79e58bf6d2319857c (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , RELEASE=main-7498493, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True) 2024-09-17T12:04:14.753 INFO:teuthology.orchestra.run.smithi045.stdout: 5 187087 9.72 MB/sec execute 375 sec latency 65.614 ms 2024-09-17T12:04:14.989 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:14 smithi045 podman[73546]: 2024-09-17 12:04:14.897562054 +0000 UTC m=+0.511360202 container init e13c2ecd369ac53b2f9288427f2a071d5c802d3effe0cbc79e58bf6d2319857c (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=) 2024-09-17T12:04:14.989 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:14 smithi045 podman[73546]: 2024-09-17 12:04:14.918626142 +0000 UTC m=+0.532424272 container start e13c2ecd369ac53b2f9288427f2a071d5c802d3effe0cbc79e58bf6d2319857c (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=main-7498493) 2024-09-17T12:04:14.989 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:14 smithi045 ceph-osd[73587]: -- 172.21.15.45:0/698957314 <== mon.1 v2:172.21.15.165:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55583b58c1e0 con 0x55583b582000 2024-09-17T12:04:14.991 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:04:14.991 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T12:04:14.991 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T12:04:14.992 INFO:teuthology.orchestra.run.smithi045.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T12:04:14.992 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [ 2024-09-17T12:04:14.992 INFO:teuthology.orchestra.run.smithi045.stdout: "crash", 2024-09-17T12:04:14.992 INFO:teuthology.orchestra.run.smithi045.stdout: "mon", 2024-09-17T12:04:14.992 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr" 2024-09-17T12:04:14.992 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-17T12:04:14.992 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "8/23 daemons upgraded", 2024-09-17T12:04:14.992 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T12:04:14.992 INFO:teuthology.orchestra.run.smithi045.stdout: "is_paused": false 2024-09-17T12:04:14.992 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:04:15.246 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:15 smithi045 ceph-mon[63880]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:04:15.246 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:15 smithi045 ceph-mon[63880]: from='client.34288 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:04:15.246 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:15 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/3110364264' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:04:15.246 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:15 smithi045 bash[73546]: e13c2ecd369ac53b2f9288427f2a071d5c802d3effe0cbc79e58bf6d2319857c 2024-09-17T12:04:15.247 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:15 smithi045 systemd[1]: Started Ceph osd.2 for a12537de-74ea-11ef-bceb-c7b262605968. 2024-09-17T12:04:15.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:15 smithi165 ceph-mon[49373]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:04:15.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:15 smithi165 ceph-mon[49373]: from='client.34288 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:04:15.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:15 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/3110364264' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:04:15.729 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 141/1545 objects degraded (9.126%), 19 pgs degraded 2024-09-17T12:04:15.729 INFO:teuthology.orchestra.run.smithi045.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T12:04:15.729 INFO:teuthology.orchestra.run.smithi045.stdout: osd.2 (root=default,host=smithi045) is down 2024-09-17T12:04:15.729 INFO:teuthology.orchestra.run.smithi045.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 141/1545 objects degraded (9.126%), 19 pgs degraded 2024-09-17T12:04:15.729 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-09-17T12:04:15.729 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-09-17T12:04:15.729 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-09-17T12:04:15.730 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.c is active+undersized+degraded, acting [5,0] 2024-09-17T12:04:15.730 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.e is active+undersized+degraded, acting [3,7] 2024-09-17T12:04:15.730 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-09-17T12:04:15.730 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-09-17T12:04:15.730 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-09-17T12:04:15.730 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-09-17T12:04:15.730 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-09-17T12:04:15.730 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-09-17T12:04:15.730 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-09-17T12:04:15.730 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-09-17T12:04:15.730 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-09-17T12:04:15.730 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-09-17T12:04:15.730 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-09-17T12:04:15.731 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-09-17T12:04:15.731 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-09-17T12:04:15.731 INFO:teuthology.orchestra.run.smithi045.stdout: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-09-17T12:04:15.753 INFO:teuthology.orchestra.run.smithi045.stdout: 5 187422 9.71 MB/sec execute 376 sec latency 64.818 ms 2024-09-17T12:04:16.276 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:16 smithi045 ceph-mon[63880]: pgmap v129: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.0 MiB/s wr, 612 op/s; 141/1545 objects degraded (9.126%) 2024-09-17T12:04:16.276 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:16 smithi045 ceph-mon[63880]: from='client.44217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:04:16.277 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:16 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 141/1545 objects degraded (9.126%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T12:04:16.277 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:16 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:16.277 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:16 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:16.277 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:16 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:04:16.277 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:16 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/1427385269' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:04:16.416 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:16 smithi165 ceph-mon[49373]: pgmap v129: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.8 KiB/s rd, 6.0 MiB/s wr, 612 op/s; 141/1545 objects degraded (9.126%) 2024-09-17T12:04:16.417 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:16 smithi165 ceph-mon[49373]: from='client.44217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:04:16.417 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:16 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 141/1545 objects degraded (9.126%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T12:04:16.417 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:16 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:16.417 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:16 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:16.417 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:16 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:04:16.417 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:16 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/1427385269' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:04:16.753 INFO:teuthology.orchestra.run.smithi045.stdout: 5 187697 9.72 MB/sec execute 377 sec latency 74.309 ms 2024-09-17T12:04:17.290 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:17 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2[73583]: 2024-09-17T12:04:17.042+0000 7ff29de17640 -1 Falling back to public interface 2024-09-17T12:04:17.753 INFO:teuthology.orchestra.run.smithi045.stdout: 5 188061 9.72 MB/sec execute 378 sec latency 48.678 ms 2024-09-17T12:04:18.047 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:18 smithi045 ceph-mon[63880]: pgmap v130: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.3 MiB/s wr, 543 op/s; 142/1563 objects degraded (9.085%) 2024-09-17T12:04:18.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:18 smithi165 ceph-mon[49373]: pgmap v130: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.3 MiB/s wr, 543 op/s; 142/1563 objects degraded (9.085%) 2024-09-17T12:04:18.754 INFO:teuthology.orchestra.run.smithi045.stdout: 5 188523 9.72 MB/sec execute 379 sec latency 51.090 ms 2024-09-17T12:04:19.753 INFO:teuthology.orchestra.run.smithi045.stdout: 5 188913 9.73 MB/sec execute 380 sec latency 32.330 ms 2024-09-17T12:04:20.079 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:20 smithi045 ceph-mon[63880]: pgmap v131: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.0 MiB/s wr, 385 op/s; 152/1644 objects degraded (9.246%) 2024-09-17T12:04:20.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:20 smithi165 ceph-mon[49373]: pgmap v131: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.0 MiB/s wr, 385 op/s; 152/1644 objects degraded (9.246%) 2024-09-17T12:04:20.753 INFO:teuthology.orchestra.run.smithi045.stdout: 5 189218 9.72 MB/sec execute 381 sec latency 49.603 ms 2024-09-17T12:04:21.753 INFO:teuthology.orchestra.run.smithi045.stdout: 5 189544 9.71 MB/sec execute 382 sec latency 58.287 ms 2024-09-17T12:04:22.097 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:22 smithi045 ceph-mon[63880]: pgmap v132: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.5 MiB/s wr, 357 op/s; 152/1644 objects degraded (9.246%) 2024-09-17T12:04:22.097 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:22 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:22.098 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:22 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:04:22.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:22 smithi165 ceph-mon[49373]: pgmap v132: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.5 MiB/s wr, 357 op/s; 152/1644 objects degraded (9.246%) 2024-09-17T12:04:22.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:22 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:22.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:22 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:04:22.754 INFO:teuthology.orchestra.run.smithi045.stdout: 5 189859 9.70 MB/sec execute 383 sec latency 41.758 ms 2024-09-17T12:04:23.126 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:23 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 152/1656 objects degraded (9.179%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T12:04:23.126 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:22 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2[73583]: 2024-09-17T12:04:22.869+0000 7ff29de17640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-17T12:04:23.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:23 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 152/1656 objects degraded (9.179%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T12:04:23.753 INFO:teuthology.orchestra.run.smithi045.stdout: 5 190165 9.70 MB/sec execute 384 sec latency 42.093 ms 2024-09-17T12:04:24.041 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:23 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2[73583]: 2024-09-17T12:04:23.797+0000 7ff29de17640 -1 osd.2 94 log_to_monitors true 2024-09-17T12:04:24.298 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:04:24 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2[73583]: 2024-09-17T12:04:24.143+0000 7ff295bc1640 -1 osd.2 94 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T12:04:24.298 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:24 smithi045 ceph-mon[63880]: pgmap v133: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.3 MiB/s wr, 569 op/s; 152/1656 objects degraded (9.179%) 2024-09-17T12:04:24.298 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:24 smithi045 ceph-mon[63880]: from='osd.2 [v2:172.21.15.45:6818/1481634196,v1:172.21.15.45:6819/1481634196]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-17T12:04:24.298 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:24 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:24.298 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:24 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:24.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:24 smithi165 ceph-mon[49373]: pgmap v133: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 347 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.3 MiB/s wr, 569 op/s; 152/1656 objects degraded (9.179%) 2024-09-17T12:04:24.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:24 smithi165 ceph-mon[49373]: from='osd.2 [v2:172.21.15.45:6818/1481634196,v1:172.21.15.45:6819/1481634196]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-17T12:04:24.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:24 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:24.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:24 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:24.754 INFO:teuthology.orchestra.run.smithi045.stdout: 5 190745 9.73 MB/sec execute 385 sec latency 64.859 ms 2024-09-17T12:04:25.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:25 smithi045 ceph-mon[63880]: from='osd.2 [v2:172.21.15.45:6818/1481634196,v1:172.21.15.45:6819/1481634196]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T12:04:25.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:25 smithi045 ceph-mon[63880]: osdmap e97: 8 total, 7 up, 8 in 2024-09-17T12:04:25.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:25 smithi045 ceph-mon[63880]: from='osd.2 [v2:172.21.15.45:6818/1481634196,v1:172.21.15.45:6819/1481634196]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi045", "root=default"]} : dispatch 2024-09-17T12:04:25.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:25.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:25.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:25 smithi165 ceph-mon[49373]: from='osd.2 [v2:172.21.15.45:6818/1481634196,v1:172.21.15.45:6819/1481634196]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T12:04:25.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:25 smithi165 ceph-mon[49373]: osdmap e97: 8 total, 7 up, 8 in 2024-09-17T12:04:25.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:25 smithi165 ceph-mon[49373]: from='osd.2 [v2:172.21.15.45:6818/1481634196,v1:172.21.15.45:6819/1481634196]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi045", "root=default"]} : dispatch 2024-09-17T12:04:25.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:25.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:25.754 INFO:teuthology.orchestra.run.smithi045.stdout: 5 190970 9.72 MB/sec execute 386 sec latency 654.478 ms 2024-09-17T12:04:26.502 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:26 smithi045 ceph-mon[63880]: pgmap v135: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.9 MiB/s wr, 481 op/s; 149/1629 objects degraded (9.147%) 2024-09-17T12:04:26.502 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:26 smithi045 ceph-mon[63880]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T12:04:26.502 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:26 smithi045 ceph-mon[63880]: osd.2 [v2:172.21.15.45:6818/1481634196,v1:172.21.15.45:6819/1481634196] boot 2024-09-17T12:04:26.502 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:26 smithi045 ceph-mon[63880]: osdmap e98: 8 total, 8 up, 8 in 2024-09-17T12:04:26.502 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:26 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-17T12:04:26.502 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:26 smithi045 ceph-mon[63880]: osdmap e99: 8 total, 8 up, 8 in 2024-09-17T12:04:26.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:26 smithi165 ceph-mon[49373]: pgmap v135: 97 pgs: 6 active+undersized, 19 active+undersized+degraded, 72 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.9 MiB/s wr, 481 op/s; 149/1629 objects degraded (9.147%) 2024-09-17T12:04:26.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:26 smithi165 ceph-mon[49373]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T12:04:26.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:26 smithi165 ceph-mon[49373]: osd.2 [v2:172.21.15.45:6818/1481634196,v1:172.21.15.45:6819/1481634196] boot 2024-09-17T12:04:26.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:26 smithi165 ceph-mon[49373]: osdmap e98: 8 total, 8 up, 8 in 2024-09-17T12:04:26.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:26 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-17T12:04:26.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:26 smithi165 ceph-mon[49373]: osdmap e99: 8 total, 8 up, 8 in 2024-09-17T12:04:26.753 INFO:teuthology.orchestra.run.smithi045.stdout: 5 190970 9.69 MB/sec execute 387 sec latency 1654.506 ms 2024-09-17T12:04:27.754 INFO:teuthology.orchestra.run.smithi045.stdout: 5 191166 9.68 MB/sec execute 388 sec latency 2181.550 ms 2024-09-17T12:04:28.117 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:28 smithi045 ceph-mon[63880]: pgmap v138: 97 pgs: 1 remapped+peering, 5 active+undersized, 17 active+undersized+degraded, 74 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.5 MiB/s wr, 465 op/s; 122/1623 objects degraded (7.517%); 0 B/s, 2 objects/s recovering 2024-09-17T12:04:28.117 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:28 smithi045 ceph-mon[63880]: osdmap e100: 8 total, 8 up, 8 in 2024-09-17T12:04:28.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:28 smithi165 ceph-mon[49373]: pgmap v138: 97 pgs: 1 remapped+peering, 5 active+undersized, 17 active+undersized+degraded, 74 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.5 MiB/s wr, 465 op/s; 122/1623 objects degraded (7.517%); 0 B/s, 2 objects/s recovering 2024-09-17T12:04:28.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:28 smithi165 ceph-mon[49373]: osdmap e100: 8 total, 8 up, 8 in 2024-09-17T12:04:28.754 INFO:teuthology.orchestra.run.smithi045.stdout: 5 191556 9.68 MB/sec execute 389 sec latency 423.533 ms 2024-09-17T12:04:29.146 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:29 smithi045 ceph-mon[63880]: osdmap e101: 8 total, 8 up, 8 in 2024-09-17T12:04:29.146 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:29 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 122/1623 objects degraded (7.517%), 17 pgs degraded (PG_DEGRADED) 2024-09-17T12:04:29.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:29 smithi165 ceph-mon[49373]: osdmap e101: 8 total, 8 up, 8 in 2024-09-17T12:04:29.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:29 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 122/1623 objects degraded (7.517%), 17 pgs degraded (PG_DEGRADED) 2024-09-17T12:04:29.754 INFO:teuthology.orchestra.run.smithi045.stdout: 5 191896 9.68 MB/sec execute 390 sec latency 804.524 ms 2024-09-17T12:04:30.158 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:30 smithi045 ceph-mon[63880]: pgmap v141: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 6 remapped+peering, 2 active+undersized, 4 active+undersized+degraded, 84 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 5.4 MiB/s wr, 488 op/s; 47/1674 objects degraded (2.808%); 24 KiB/s, 9 objects/s recovering 2024-09-17T12:04:30.158 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:30 smithi045 ceph-mon[63880]: osdmap e102: 8 total, 8 up, 8 in 2024-09-17T12:04:30.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:30 smithi165 ceph-mon[49373]: pgmap v141: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 6 remapped+peering, 2 active+undersized, 4 active+undersized+degraded, 84 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 5.4 MiB/s wr, 488 op/s; 47/1674 objects degraded (2.808%); 24 KiB/s, 9 objects/s recovering 2024-09-17T12:04:30.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:30 smithi165 ceph-mon[49373]: osdmap e102: 8 total, 8 up, 8 in 2024-09-17T12:04:30.754 INFO:teuthology.orchestra.run.smithi045.stdout: 5 192242 9.68 MB/sec execute 391 sec latency 816.025 ms 2024-09-17T12:04:31.171 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:31 smithi045 ceph-mon[63880]: osdmap e103: 8 total, 8 up, 8 in 2024-09-17T12:04:31.171 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:31 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:31.171 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:31 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:31.171 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:31 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:04:31.171 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:31 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:04:31.172 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:31 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:31.172 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:31 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:31.172 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:31 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:04:31.172 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:31 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:04:31.172 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:31 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:04:31.172 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:31 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:04:31.172 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:31 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-17T12:04:31.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:31 smithi165 ceph-mon[49373]: osdmap e103: 8 total, 8 up, 8 in 2024-09-17T12:04:31.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:31 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:31.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:31 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:31.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:31 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:04:31.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:31 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:04:31.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:31 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:31.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:31 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:31.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:31 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:04:31.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:31 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:04:31.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:31 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:04:31.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:31 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:04:31.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:31 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-17T12:04:31.754 INFO:teuthology.orchestra.run.smithi045.stdout: 5 192707 9.68 MB/sec execute 392 sec latency 60.728 ms 2024-09-17T12:04:32.175 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:32 smithi045 ceph-mon[63880]: pgmap v144: 97 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 6 remapped+peering, 88 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.0 MiB/s wr, 469 op/s; 20/1674 objects degraded (1.195%); 24 KiB/s, 5 objects/s recovering 2024-09-17T12:04:32.175 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:32 smithi045 ceph-mon[63880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T12:04:32.175 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:32 smithi045 ceph-mon[63880]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-17T12:04:32.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:32 smithi165 ceph-mon[49373]: pgmap v144: 97 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 6 remapped+peering, 88 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.0 MiB/s wr, 469 op/s; 20/1674 objects degraded (1.195%); 24 KiB/s, 5 objects/s recovering 2024-09-17T12:04:32.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:32 smithi165 ceph-mon[49373]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T12:04:32.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:32 smithi165 ceph-mon[49373]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-17T12:04:32.754 INFO:teuthology.orchestra.run.smithi045.stdout: 5 193133 9.68 MB/sec execute 393 sec latency 51.067 ms 2024-09-17T12:04:33.754 INFO:teuthology.orchestra.run.smithi045.stdout: 5 193567 9.69 MB/sec execute 394 sec latency 31.942 ms 2024-09-17T12:04:34.226 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:34 smithi045 ceph-mon[63880]: pgmap v145: 97 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.3 MiB/s wr, 372 op/s; 20/1626 objects degraded (1.230%); 711 KiB/s, 21 objects/s recovering 2024-09-17T12:04:34.226 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:34 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 20/1626 objects degraded (1.230%), 1 pg degraded (PG_DEGRADED) 2024-09-17T12:04:34.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:34 smithi165 ceph-mon[49373]: pgmap v145: 97 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.3 MiB/s wr, 372 op/s; 20/1626 objects degraded (1.230%); 711 KiB/s, 21 objects/s recovering 2024-09-17T12:04:34.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:34 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 20/1626 objects degraded (1.230%), 1 pg degraded (PG_DEGRADED) 2024-09-17T12:04:34.754 INFO:teuthology.orchestra.run.smithi045.stdout: 5 193925 9.70 MB/sec execute 395 sec latency 52.111 ms 2024-09-17T12:04:35.267 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:35 smithi045 ceph-mon[63880]: pgmap v146: 97 pgs: 97 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.9 MiB/s wr, 507 op/s; 1.9 MiB/s, 21 objects/s recovering 2024-09-17T12:04:35.267 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:35 smithi045 ceph-mon[63880]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/1626 objects degraded (1.230%), 1 pg degraded) 2024-09-17T12:04:35.267 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:35 smithi045 ceph-mon[63880]: Cluster is now healthy 2024-09-17T12:04:35.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:35 smithi165 ceph-mon[49373]: pgmap v146: 97 pgs: 97 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.9 MiB/s wr, 507 op/s; 1.9 MiB/s, 21 objects/s recovering 2024-09-17T12:04:35.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:35 smithi165 ceph-mon[49373]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/1626 objects degraded (1.230%), 1 pg degraded) 2024-09-17T12:04:35.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:35 smithi165 ceph-mon[49373]: Cluster is now healthy 2024-09-17T12:04:35.754 INFO:teuthology.orchestra.run.smithi045.stdout: 5 194258 9.70 MB/sec execute 396 sec latency 40.541 ms 2024-09-17T12:04:36.754 INFO:teuthology.orchestra.run.smithi045.stdout: 5 194560 9.70 MB/sec execute 397 sec latency 59.700 ms 2024-09-17T12:04:37.571 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:37 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:37.571 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:37 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:04:37.571 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:37 smithi045 ceph-mon[63880]: pgmap v147: 97 pgs: 97 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.0 MiB/s wr, 427 op/s; 1.6 MiB/s, 17 objects/s recovering 2024-09-17T12:04:37.762 INFO:teuthology.orchestra.run.smithi045.stdout: 5 194901 9.70 MB/sec execute 398 sec latency 36.943 ms 2024-09-17T12:04:37.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:37 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:37.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:37 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:04:37.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:37 smithi165 ceph-mon[49373]: pgmap v147: 97 pgs: 97 active+clean; 348 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.0 MiB/s wr, 427 op/s; 1.6 MiB/s, 17 objects/s recovering 2024-09-17T12:04:38.762 INFO:teuthology.orchestra.run.smithi045.stdout: 5 195263 9.69 MB/sec execute 399 sec latency 36.137 ms 2024-09-17T12:04:39.762 INFO:teuthology.orchestra.run.smithi045.stdout: 5 195612 9.69 MB/sec execute 400 sec latency 37.030 ms 2024-09-17T12:04:40.113 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:39 smithi045 ceph-mon[63880]: pgmap v148: 97 pgs: 97 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.7 MiB/s wr, 554 op/s; 1.3 MiB/s, 14 objects/s recovering 2024-09-17T12:04:40.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:39 smithi165 ceph-mon[49373]: pgmap v148: 97 pgs: 97 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.7 MiB/s wr, 554 op/s; 1.3 MiB/s, 14 objects/s recovering 2024-09-17T12:04:40.762 INFO:teuthology.orchestra.run.smithi045.stdout: 5 195900 9.69 MB/sec execute 401 sec latency 50.183 ms 2024-09-17T12:04:41.762 INFO:teuthology.orchestra.run.smithi045.stdout: 5 196225 9.68 MB/sec execute 402 sec latency 99.648 ms 2024-09-17T12:04:42.183 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:41 smithi045 ceph-mon[63880]: pgmap v149: 97 pgs: 97 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.2 MiB/s wr, 505 op/s; 1.2 MiB/s, 13 objects/s recovering 2024-09-17T12:04:42.183 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:41 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:42.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:41 smithi165 ceph-mon[49373]: pgmap v149: 97 pgs: 97 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.2 MiB/s wr, 505 op/s; 1.2 MiB/s, 13 objects/s recovering 2024-09-17T12:04:42.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:41 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:42.762 INFO:teuthology.orchestra.run.smithi045.stdout: 5 196547 9.67 MB/sec execute 403 sec latency 68.474 ms 2024-09-17T12:04:43.762 INFO:teuthology.orchestra.run.smithi045.stdout: 5 196941 9.67 MB/sec execute 404 sec latency 46.347 ms 2024-09-17T12:04:43.973 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:43 smithi045 ceph-mon[63880]: pgmap v150: 97 pgs: 97 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.3 MiB/s wr, 646 op/s; 1.1 MiB/s, 11 objects/s recovering 2024-09-17T12:04:44.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:43 smithi165 ceph-mon[49373]: pgmap v150: 97 pgs: 97 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 6.3 MiB/s wr, 646 op/s; 1.1 MiB/s, 11 objects/s recovering 2024-09-17T12:04:44.762 INFO:teuthology.orchestra.run.smithi045.stdout: 5 197539 9.70 MB/sec execute 405 sec latency 37.152 ms 2024-09-17T12:04:45.762 INFO:teuthology.orchestra.run.smithi045.stdout: 5 198158 9.72 MB/sec execute 406 sec latency 33.460 ms 2024-09-17T12:04:45.993 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:45 smithi045 ceph-mon[63880]: pgmap v151: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.9 MiB/s wr, 495 op/s; 760 KiB/s, 1 objects/s recovering 2024-09-17T12:04:45.993 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-17T12:04:46.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:45 smithi165 ceph-mon[49373]: pgmap v151: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.9 MiB/s wr, 495 op/s; 760 KiB/s, 1 objects/s recovering 2024-09-17T12:04:46.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-17T12:04:46.307 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T12:04:46.762 INFO:teuthology.orchestra.run.smithi045.stdout: 5 198489 9.72 MB/sec execute 407 sec latency 54.078 ms 2024-09-17T12:04:46.998 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:46 smithi045 ceph-mon[63880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T12:04:46.998 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:46 smithi045 ceph-mon[63880]: Upgrade: osd.3 is safe to restart 2024-09-17T12:04:47.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:46 smithi165 ceph-mon[49373]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T12:04:47.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:46 smithi165 ceph-mon[49373]: Upgrade: osd.3 is safe to restart 2024-09-17T12:04:47.379 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T12:04:47.379 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (6m) 23s ago 15m 20.3M - 0.27.0 11f11916f8cd 2a2a54402d2a 2024-09-17T12:04:47.379 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (2m) 23s ago 15m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 e545cbf7736d 2024-09-17T12:04:47.379 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (2m) 2m ago 14m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 c1e89e114e4d 2024-09-17T12:04:47.379 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (5m) 23s ago 15m 57.0M - dad864ee21e9 371aaf55379f 2024-09-17T12:04:47.379 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (9m) 23s ago 9m 37.6M - 17.2.0 e1d6a67b021e 6a2a4a998dd1 2024-09-17T12:04:47.379 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (9m) 2m ago 9m 16.0M - 17.2.0 e1d6a67b021e d4c2987b9bac 2024-09-17T12:04:47.379 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:8443,9283,8765 running (5m) 23s ago 16m 519M - 19.3.0-4937-g74984932 6c1fcab85602 ae2f507bd67d 2024-09-17T12:04:47.379 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283,8765 running (4m) 2m ago 14m 453M - 19.3.0-4937-g74984932 6c1fcab85602 53403866652e 2024-09-17T12:04:47.379 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (4m) 23s ago 16m 51.5M 2048M 19.3.0-4937-g74984932 6c1fcab85602 737eec78e0f0 2024-09-17T12:04:47.379 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (3m) 2m ago 14m 36.1M 2048M 19.3.0-4937-g74984932 6c1fcab85602 249eedb4123e 2024-09-17T12:04:47.379 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (9m) 23s ago 9m 280M - 4.0 e1d6a67b021e b8fc749ebb71 2024-09-17T12:04:47.380 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (8m) 2m ago 8m 83.8M - 4.0 e1d6a67b021e 771b27764432 2024-09-17T12:04:47.380 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (5m) 23s ago 15m 11.1M - 1.7.0 72c9c2088986 989b5738cb69 2024-09-17T12:04:47.380 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (5m) 2m ago 14m 11.9M - 1.7.0 72c9c2088986 e1e8ffab3f21 2024-09-17T12:04:47.380 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (2m) 23s ago 13m 281M 1937M 19.3.0-4937-g74984932 6c1fcab85602 20dafd01bd12 2024-09-17T12:04:47.380 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (81s) 23s ago 13m 224M 1937M 19.3.0-4937-g74984932 6c1fcab85602 d39be5014418 2024-09-17T12:04:47.380 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (32s) 23s ago 12m 12.1M 1937M 19.3.0-4937-g74984932 6c1fcab85602 e13c2ecd369a 2024-09-17T12:04:47.380 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (12m) 23s ago 12m 604M 1937M 17.2.0 e1d6a67b021e 92a2389488e4 2024-09-17T12:04:47.380 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (12m) 2m ago 12m 494M 2704M 17.2.0 e1d6a67b021e e29eb74df25a 2024-09-17T12:04:47.380 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (11m) 2m ago 11m 544M 2704M 17.2.0 e1d6a67b021e 5d900eb9165b 2024-09-17T12:04:47.380 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (11m) 2m ago 11m 466M 2704M 17.2.0 e1d6a67b021e fae3e1bae466 2024-09-17T12:04:47.380 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (10m) 2m ago 10m 371M 2704M 17.2.0 e1d6a67b021e 71b9f2cdb810 2024-09-17T12:04:47.380 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (4m) 23s ago 14m 56.0M - 2.51.0 1d3b7f56885b 61b064ad6609 2024-09-17T12:04:47.769 INFO:teuthology.orchestra.run.smithi045.stdout: 5 198966 9.72 MB/sec execute 408 sec latency 51.106 ms 2024-09-17T12:04:48.004 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:47 smithi045 ceph-mon[63880]: from='client.44227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:04:48.004 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:47 smithi045 ceph-mon[63880]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:04:48.004 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:47 smithi045 ceph-mon[63880]: pgmap v152: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 400 op/s 2024-09-17T12:04:48.075 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:04:48.075 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T12:04:48.075 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:04:48.076 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:04:48.076 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T12:04:48.076 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:04:48.076 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:04:48.076 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T12:04:48.076 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-17T12:04:48.076 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 3 2024-09-17T12:04:48.076 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:04:48.076 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T12:04:48.076 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T12:04:48.076 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:04:48.076 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T12:04:48.076 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-17T12:04:48.076 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 7 2024-09-17T12:04:48.077 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T12:04:48.077 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:04:48.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:47 smithi165 ceph-mon[49373]: from='client.44227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:04:48.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:47 smithi165 ceph-mon[49373]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:04:48.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:47 smithi165 ceph-mon[49373]: pgmap v152: 97 pgs: 97 active+clean; 344 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.3 MiB/s wr, 400 op/s 2024-09-17T12:04:48.652 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:04:48.653 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T12:04:48.653 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T12:04:48.653 INFO:teuthology.orchestra.run.smithi045.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T12:04:48.653 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [ 2024-09-17T12:04:48.653 INFO:teuthology.orchestra.run.smithi045.stdout: "crash", 2024-09-17T12:04:48.653 INFO:teuthology.orchestra.run.smithi045.stdout: "mon", 2024-09-17T12:04:48.653 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr" 2024-09-17T12:04:48.653 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-17T12:04:48.653 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "9/23 daemons upgraded", 2024-09-17T12:04:48.653 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T12:04:48.653 INFO:teuthology.orchestra.run.smithi045.stdout: "is_paused": false 2024-09-17T12:04:48.653 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:04:48.769 INFO:teuthology.orchestra.run.smithi045.stdout: 5 199341 9.73 MB/sec execute 409 sec latency 53.677 ms 2024-09-17T12:04:49.018 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:48 smithi045 ceph-mon[63880]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:04:49.018 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:48 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/924934690' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:04:49.018 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:48 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:49.018 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:48 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-17T12:04:49.018 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:48 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:04:49.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:48 smithi165 ceph-mon[49373]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:04:49.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:48 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/924934690' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:04:49.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:48 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:04:49.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:48 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-17T12:04:49.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:48 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:04:49.361 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_OK 2024-09-17T12:04:49.769 INFO:teuthology.orchestra.run.smithi045.stdout: 5 199751 9.72 MB/sec execute 410 sec latency 53.330 ms 2024-09-17T12:04:50.041 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:49 smithi045 ceph-mon[63880]: Upgrade: Updating osd.3 2024-09-17T12:04:50.041 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:49 smithi045 ceph-mon[63880]: Deploying daemon osd.3 on smithi045 2024-09-17T12:04:50.041 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:49 smithi045 ceph-mon[63880]: from='client.34322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:04:50.041 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:49 smithi045 ceph-mon[63880]: pgmap v153: 97 pgs: 97 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 6.8 MiB/s wr, 647 op/s 2024-09-17T12:04:50.041 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:49 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/3123548501' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:04:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:49 smithi165 ceph-mon[49373]: Upgrade: Updating osd.3 2024-09-17T12:04:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:49 smithi165 ceph-mon[49373]: Deploying daemon osd.3 on smithi045 2024-09-17T12:04:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:49 smithi165 ceph-mon[49373]: from='client.34322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:04:50.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:49 smithi165 ceph-mon[49373]: pgmap v153: 97 pgs: 97 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 6.8 MiB/s wr, 647 op/s 2024-09-17T12:04:50.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:49 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/3123548501' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:04:50.769 INFO:teuthology.orchestra.run.smithi045.stdout: 5 200216 9.72 MB/sec execute 411 sec latency 36.581 ms 2024-09-17T12:04:51.769 INFO:teuthology.orchestra.run.smithi045.stdout: 5 200661 9.73 MB/sec execute 412 sec latency 34.695 ms 2024-09-17T12:04:52.078 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:51 smithi045 ceph-mon[63880]: pgmap v154: 97 pgs: 97 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.8 MiB/s wr, 480 op/s 2024-09-17T12:04:52.078 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:04:52.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:51 smithi165 ceph-mon[49373]: pgmap v154: 97 pgs: 97 active+clean; 350 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.8 MiB/s wr, 480 op/s 2024-09-17T12:04:52.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:51 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:04:52.769 INFO:teuthology.orchestra.run.smithi045.stdout: 5 201085 9.74 MB/sec execute 413 sec latency 45.893 ms 2024-09-17T12:04:53.598 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:04:53 smithi045 systemd[1]: Stopping Ceph osd.3 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:04:53.769 INFO:teuthology.orchestra.run.smithi045.stdout: 5 201524 9.75 MB/sec execute 414 sec latency 51.365 ms 2024-09-17T12:04:54.167 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:53 smithi045 ceph-mon[63880]: pgmap v155: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.9 MiB/s wr, 726 op/s 2024-09-17T12:04:54.167 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:04:54 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3[44644]: 2024-09-17T12:04:54.010+0000 7fe138de6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T12:04:54.167 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:04:54 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3[44644]: 2024-09-17T12:04:54.010+0000 7fe138de6700 -1 osd.3 103 *** Got signal Terminated *** 2024-09-17T12:04:54.167 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:04:54 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3[44644]: 2024-09-17T12:04:54.010+0000 7fe138de6700 -1 osd.3 103 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T12:04:54.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:53 smithi165 ceph-mon[49373]: pgmap v155: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.9 MiB/s wr, 726 op/s 2024-09-17T12:04:54.769 INFO:teuthology.orchestra.run.smithi045.stdout: 5 201970 9.76 MB/sec execute 415 sec latency 27.354 ms 2024-09-17T12:04:55.181 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:54 smithi045 ceph-mon[63880]: osd.3 marked itself down and dead 2024-09-17T12:04:55.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:54 smithi165 ceph-mon[49373]: osd.3 marked itself down and dead 2024-09-17T12:04:55.769 INFO:teuthology.orchestra.run.smithi045.stdout: 5 202236 9.74 MB/sec execute 416 sec latency 682.548 ms 2024-09-17T12:04:55.959 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:04:55 smithi045 podman[75492]: 2024-09-17 12:04:55.687410259 +0000 UTC m=+1.864605604 container died 92a2389488e45c22e5197525453d090a9e007f372b4c0ac81aa36e6f69e71bdb (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, release=754, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, vcs-type=git, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, name=centos-stream, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-17T12:04:56.211 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:55 smithi045 ceph-mon[63880]: pgmap v156: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.8 MiB/s wr, 557 op/s 2024-09-17T12:04:56.211 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:55 smithi045 ceph-mon[63880]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T12:04:56.211 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:55 smithi045 ceph-mon[63880]: osdmap e104: 8 total, 7 up, 8 in 2024-09-17T12:04:56.211 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:04:56 smithi045 podman[75492]: 2024-09-17 12:04:56.074171548 +0000 UTC m=+2.251366883 container cleanup 92a2389488e45c22e5197525453d090a9e007f372b4c0ac81aa36e6f69e71bdb (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, version=8, vcs-type=git, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, ceph=True, io.openshift.tags=base centos centos-stream, release=754, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, maintainer=Guillaume Abrioux , io.openshift.expose-services=, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-17T12:04:56.211 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:04:56 smithi045 bash[75492]: ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3 2024-09-17T12:04:56.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:55 smithi165 ceph-mon[49373]: pgmap v156: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.8 MiB/s wr, 557 op/s 2024-09-17T12:04:56.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:55 smithi165 ceph-mon[49373]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T12:04:56.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:55 smithi165 ceph-mon[49373]: osdmap e104: 8 total, 7 up, 8 in 2024-09-17T12:04:56.721 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:04:56 smithi045 podman[75506]: 2024-09-17 12:04:56.469245461 +0000 UTC m=+0.780861822 container remove 92a2389488e45c22e5197525453d090a9e007f372b4c0ac81aa36e6f69e71bdb (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, version=8, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, distribution-scope=public, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-17T12:04:56.769 INFO:teuthology.orchestra.run.smithi045.stdout: 5 202556 9.73 MB/sec execute 417 sec latency 953.300 ms 2024-09-17T12:04:57.231 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:56 smithi045 ceph-mon[63880]: osdmap e105: 8 total, 7 up, 8 in 2024-09-17T12:04:57.231 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:04:57 smithi045 podman[75591]: 2024-09-17 12:04:56.943846306 +0000 UTC m=+0.034152469 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:04:57.231 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:04:57 smithi045 podman[75591]: 2024-09-17 12:04:57.132854563 +0000 UTC m=+0.223160713 container create 8d0d3eaaaedd9092676c66fc9149810576b8185e15298e408e1894a2c5169ccf (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-deactivate, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-17T12:04:57.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:56 smithi165 ceph-mon[49373]: osdmap e105: 8 total, 7 up, 8 in 2024-09-17T12:04:57.739 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:04:57 smithi045 podman[75591]: 2024-09-17 12:04:57.549741203 +0000 UTC m=+0.640047373 container init 8d0d3eaaaedd9092676c66fc9149810576b8185e15298e408e1894a2c5169ccf (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=main-7498493, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD) 2024-09-17T12:04:57.739 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:04:57 smithi045 podman[75591]: 2024-09-17 12:04:57.564965068 +0000 UTC m=+0.655271225 container start 8d0d3eaaaedd9092676c66fc9149810576b8185e15298e408e1894a2c5169ccf (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-deactivate, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=main-7498493) 2024-09-17T12:04:57.769 INFO:teuthology.orchestra.run.smithi045.stdout: 5 203062 9.76 MB/sec execute 418 sec latency 52.738 ms 2024-09-17T12:04:58.004 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:04:57 smithi045 podman[75591]: 2024-09-17 12:04:57.767229742 +0000 UTC m=+0.857535910 container attach 8d0d3eaaaedd9092676c66fc9149810576b8185e15298e408e1894a2c5169ccf (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-17T12:04:58.004 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:04:57 smithi045 podman[75591]: 2024-09-17 12:04:57.950679678 +0000 UTC m=+1.040985849 container died 8d0d3eaaaedd9092676c66fc9149810576b8185e15298e408e1894a2c5169ccf (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-17T12:04:58.254 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:58 smithi045 ceph-mon[63880]: pgmap v159: 97 pgs: 4 active+undersized, 16 stale+active+clean, 77 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.3 MiB/s wr, 425 op/s 2024-09-17T12:04:58.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:58 smithi165 ceph-mon[49373]: pgmap v159: 97 pgs: 4 active+undersized, 16 stale+active+clean, 77 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.3 MiB/s wr, 425 op/s 2024-09-17T12:04:58.769 INFO:teuthology.orchestra.run.smithi045.stdout: 5 203455 9.75 MB/sec execute 419 sec latency 48.204 ms 2024-09-17T12:04:59.026 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:04:58 smithi045 podman[75612]: 2024-09-17 12:04:58.902254195 +0000 UTC m=+0.944792758 container remove 8d0d3eaaaedd9092676c66fc9149810576b8185e15298e408e1894a2c5169ccf (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-17T12:04:59.026 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:04:58 smithi045 systemd[1]: ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-17T12:04:59.026 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:04:58 smithi045 systemd[1]: Stopped Ceph osd.3 for a12537de-74ea-11ef-bceb-c7b262605968. 2024-09-17T12:04:59.027 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:04:58 smithi045 systemd[1]: ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.3.service: Consumed 3min 57.157s CPU time. 2024-09-17T12:04:59.295 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:04:59 smithi045 ceph-mon[63880]: Health check failed: Degraded data redundancy: 192/1620 objects degraded (11.852%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T12:04:59.295 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:04:59 smithi045 systemd[1]: Starting Ceph osd.3 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:04:59.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:04:59 smithi165 ceph-mon[49373]: Health check failed: Degraded data redundancy: 192/1620 objects degraded (11.852%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T12:04:59.560 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:04:59 smithi045 podman[75710]: 2024-09-17 12:04:59.419136137 +0000 UTC m=+0.019070254 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:04:59.769 INFO:teuthology.orchestra.run.smithi045.stdout: 5 203750 9.74 MB/sec execute 420 sec latency 53.330 ms 2024-09-17T12:04:59.839 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:04:59 smithi045 podman[75710]: 2024-09-17 12:04:59.559047166 +0000 UTC m=+0.158981270 container create ba220394f529e13695f7676ce0170110385211b8e116e20880723d9640c236f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-activate, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-17T12:05:00.107 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:00 smithi045 ceph-mon[63880]: pgmap v160: 97 pgs: 13 active+undersized, 1 stale+active+clean, 24 active+undersized+degraded, 59 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 MiB/s wr, 757 op/s; 192/1620 objects degraded (11.852%) 2024-09-17T12:05:00.107 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:04:59 smithi045 podman[75710]: 2024-09-17 12:04:59.946589179 +0000 UTC m=+0.546523296 container init ba220394f529e13695f7676ce0170110385211b8e116e20880723d9640c236f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-activate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-17T12:05:00.108 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:04:59 smithi045 podman[75710]: 2024-09-17 12:04:59.956136151 +0000 UTC m=+0.556070251 container start ba220394f529e13695f7676ce0170110385211b8e116e20880723d9640c236f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=main-7498493, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T12:05:00.359 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:00 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-activate[75724]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:05:00.359 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:00 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-activate[75724]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:05:00.359 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:00 smithi045 bash[75710]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:05:00.359 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:00 smithi045 podman[75710]: 2024-09-17 12:05:00.355530757 +0000 UTC m=+0.955464855 container attach ba220394f529e13695f7676ce0170110385211b8e116e20880723d9640c236f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, ceph=True, RELEASE=main-7498493, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-17T12:05:00.359 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:00 smithi045 bash[75710]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:05:00.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:00 smithi165 ceph-mon[49373]: pgmap v160: 97 pgs: 13 active+undersized, 1 stale+active+clean, 24 active+undersized+degraded, 59 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.7 MiB/s wr, 757 op/s; 192/1620 objects degraded (11.852%) 2024-09-17T12:05:00.770 INFO:teuthology.orchestra.run.smithi045.stdout: 5 203969 9.73 MB/sec execute 421 sec latency 60.085 ms 2024-09-17T12:05:00.895 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:00 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-activate[75724]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T12:05:00.895 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:00 smithi045 bash[75710]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T12:05:00.895 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:00 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-activate[75724]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T12:05:00.895 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:00 smithi045 bash[75710]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T12:05:01.412 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:01 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-activate[75724]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T12:05:01.412 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:01 smithi045 bash[75710]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T12:05:01.413 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:01 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-activate[75724]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T12:05:01.413 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:01 smithi045 bash[75710]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T12:05:01.413 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:01 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-activate[75724]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-17T12:05:01.413 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:01 smithi045 bash[75710]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-17T12:05:01.413 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:01 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-activate[75724]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T12:05:01.413 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:01 smithi045 bash[75710]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T12:05:01.413 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:01 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-activate[75724]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-17T12:05:01.413 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:01 smithi045 bash[75710]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-17T12:05:01.413 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:01 smithi045 podman[75710]: 2024-09-17 12:05:01.190637412 +0000 UTC m=+1.790571516 container died ba220394f529e13695f7676ce0170110385211b8e116e20880723d9640c236f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-7498493, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T12:05:01.770 INFO:teuthology.orchestra.run.smithi045.stdout: 5 204221 9.73 MB/sec execute 422 sec latency 56.625 ms 2024-09-17T12:05:02.185 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:02 smithi045 ceph-mon[63880]: pgmap v161: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.5 MiB/s wr, 388 op/s; 204/1620 objects degraded (12.593%) 2024-09-17T12:05:02.185 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:02 smithi045 podman[75868]: 2024-09-17 12:05:02.070877993 +0000 UTC m=+0.868042329 container remove ba220394f529e13695f7676ce0170110385211b8e116e20880723d9640c236f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, RELEASE=main-7498493, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T12:05:02.436 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:02 smithi045 podman[75923]: 2024-09-17 12:05:02.208850346 +0000 UTC m=+0.025321051 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:05:02.436 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:02 smithi045 podman[75923]: 2024-09-17 12:05:02.347021984 +0000 UTC m=+0.163492690 container create ffd68312d2425a78fa50437bb2a043ea84ce606a0fe2480228dfa70d1708603a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , RELEASE=main-7498493, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-17T12:05:02.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:02 smithi165 ceph-mon[49373]: pgmap v161: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.5 MiB/s wr, 388 op/s; 204/1620 objects degraded (12.593%) 2024-09-17T12:05:02.697 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:02 smithi045 podman[75923]: 2024-09-17 12:05:02.624611472 +0000 UTC m=+0.441082185 container init ffd68312d2425a78fa50437bb2a043ea84ce606a0fe2480228dfa70d1708603a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=main-7498493, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True) 2024-09-17T12:05:02.697 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:02 smithi045 podman[75923]: 2024-09-17 12:05:02.629936818 +0000 UTC m=+0.446407531 container start ffd68312d2425a78fa50437bb2a043ea84ce606a0fe2480228dfa70d1708603a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=main-7498493, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS) 2024-09-17T12:05:02.697 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:02 smithi045 ceph-osd[75958]: -- 172.21.15.45:0/987040675 <== mon.0 v2:172.21.15.45:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55bab67141e0 con 0x55bab6710000 2024-09-17T12:05:02.770 INFO:teuthology.orchestra.run.smithi045.stdout: 5 204524 9.74 MB/sec execute 423 sec latency 42.119 ms 2024-09-17T12:05:02.948 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:02 smithi045 bash[75923]: ffd68312d2425a78fa50437bb2a043ea84ce606a0fe2480228dfa70d1708603a 2024-09-17T12:05:02.948 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:02 smithi045 systemd[1]: Started Ceph osd.3 for a12537de-74ea-11ef-bceb-c7b262605968. 2024-09-17T12:05:03.770 INFO:teuthology.orchestra.run.smithi045.stdout: 5 204859 9.74 MB/sec execute 424 sec latency 72.755 ms 2024-09-17T12:05:03.973 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:03 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:03.973 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:03 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:03.973 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:03 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:05:03.973 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:03 smithi045 ceph-mon[63880]: pgmap v162: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 MiB/s wr, 626 op/s; 206/1590 objects degraded (12.956%) 2024-09-17T12:05:04.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:03 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:04.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:03 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:04.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:03 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:05:04.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:03 smithi165 ceph-mon[49373]: pgmap v162: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.2 MiB/s wr, 626 op/s; 206/1590 objects degraded (12.956%) 2024-09-17T12:05:04.479 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:04 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3[75938]: 2024-09-17T12:05:04.287+0000 7fafd0a9f640 -1 Falling back to public interface 2024-09-17T12:05:04.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 205202 9.73 MB/sec execute 425 sec latency 45.340 ms 2024-09-17T12:05:05.232 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:05 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 205/1584 objects degraded (12.942%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T12:05:05.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:05 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 205/1584 objects degraded (12.942%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T12:05:05.770 INFO:teuthology.orchestra.run.smithi045.stdout: 5 205555 9.73 MB/sec execute 426 sec latency 53.097 ms 2024-09-17T12:05:06.240 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:06 smithi045 ceph-mon[63880]: pgmap v163: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.2 MiB/s wr, 518 op/s; 205/1584 objects degraded (12.942%) 2024-09-17T12:05:06.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:06 smithi165 ceph-mon[49373]: pgmap v163: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.2 MiB/s wr, 518 op/s; 205/1584 objects degraded (12.942%) 2024-09-17T12:05:06.770 INFO:teuthology.orchestra.run.smithi045.stdout: 5 206003 9.72 MB/sec execute 427 sec latency 55.913 ms 2024-09-17T12:05:07.524 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:07 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:07.524 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:07 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:05:07.524 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:07 smithi045 ceph-mon[63880]: pgmap v164: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.9 MiB/s wr, 479 op/s; 205/1620 objects degraded (12.654%) 2024-09-17T12:05:07.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:07 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:07.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:07 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:05:07.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:07 smithi165 ceph-mon[49373]: pgmap v164: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.9 MiB/s wr, 479 op/s; 205/1620 objects degraded (12.654%) 2024-09-17T12:05:07.770 INFO:teuthology.orchestra.run.smithi045.stdout: 5 206398 9.73 MB/sec execute 428 sec latency 35.077 ms 2024-09-17T12:05:08.770 INFO:teuthology.orchestra.run.smithi045.stdout: 5 206936 9.74 MB/sec execute 429 sec latency 52.262 ms 2024-09-17T12:05:09.770 INFO:teuthology.orchestra.run.smithi045.stdout: 5 207286 9.74 MB/sec execute 430 sec latency 42.554 ms 2024-09-17T12:05:10.076 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:09 smithi045 ceph-mon[63880]: pgmap v165: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 6.4 MiB/s wr, 637 op/s; 212/1692 objects degraded (12.530%) 2024-09-17T12:05:10.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:09 smithi165 ceph-mon[49373]: pgmap v165: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 6.4 MiB/s wr, 637 op/s; 212/1692 objects degraded (12.530%) 2024-09-17T12:05:10.770 INFO:teuthology.orchestra.run.smithi045.stdout: 5 207540 9.73 MB/sec execute 431 sec latency 36.309 ms 2024-09-17T12:05:11.608 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:11 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3[75938]: 2024-09-17T12:05:11.556+0000 7fafd0a9f640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-17T12:05:11.770 INFO:teuthology.orchestra.run.smithi045.stdout: 5 207873 9.74 MB/sec execute 432 sec latency 39.961 ms 2024-09-17T12:05:12.115 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:11 smithi045 ceph-mon[63880]: pgmap v166: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 4.2 MiB/s wr, 416 op/s; 212/1692 objects degraded (12.530%) 2024-09-17T12:05:12.115 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:11 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:12.115 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:11 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:12.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:11 smithi165 ceph-mon[49373]: pgmap v166: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 4.2 MiB/s wr, 416 op/s; 212/1692 objects degraded (12.530%) 2024-09-17T12:05:12.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:11 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:12.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:11 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:12.770 INFO:teuthology.orchestra.run.smithi045.stdout: 5 208262 9.74 MB/sec execute 433 sec latency 40.127 ms 2024-09-17T12:05:13.123 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:13 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3[75938]: 2024-09-17T12:05:13.067+0000 7fafd0a9f640 -1 osd.3 103 log_to_monitors true 2024-09-17T12:05:13.645 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:13 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:13.645 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:13 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:13.645 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:13 smithi045 ceph-mon[63880]: pgmap v167: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 6.1 MiB/s wr, 619 op/s; 202/1617 objects degraded (12.492%) 2024-09-17T12:05:13.645 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:13 smithi045 ceph-mon[63880]: from='osd.3 [v2:172.21.15.45:6826/1544784093,v1:172.21.15.45:6827/1544784093]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-17T12:05:13.645 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:13 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 212/1692 objects degraded (12.530%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T12:05:13.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:13 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:13.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:13 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:13.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:13 smithi165 ceph-mon[49373]: pgmap v167: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 6.1 MiB/s wr, 619 op/s; 202/1617 objects degraded (12.492%) 2024-09-17T12:05:13.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:13 smithi165 ceph-mon[49373]: from='osd.3 [v2:172.21.15.45:6826/1544784093,v1:172.21.15.45:6827/1544784093]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-17T12:05:13.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:13 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 212/1692 objects degraded (12.530%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T12:05:13.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 208624 9.74 MB/sec execute 434 sec latency 52.198 ms 2024-09-17T12:05:13.896 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:05:13 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3[75938]: 2024-09-17T12:05:13.833+0000 7fafc4841640 -1 osd.3 103 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T12:05:14.664 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:14 smithi045 ceph-mon[63880]: from='osd.3 [v2:172.21.15.45:6826/1544784093,v1:172.21.15.45:6827/1544784093]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T12:05:14.664 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:14 smithi045 ceph-mon[63880]: osdmap e106: 8 total, 7 up, 8 in 2024-09-17T12:05:14.664 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:14 smithi045 ceph-mon[63880]: from='osd.3 [v2:172.21.15.45:6826/1544784093,v1:172.21.15.45:6827/1544784093]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi045", "root=default"]} : dispatch 2024-09-17T12:05:14.664 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:14 smithi045 ceph-mon[63880]: from='osd.3 [v2:172.21.15.45:6826/1544784093,v1:172.21.15.45:6827/1544784093]' entity='osd.3' 2024-09-17T12:05:14.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:14 smithi165 ceph-mon[49373]: from='osd.3 [v2:172.21.15.45:6826/1544784093,v1:172.21.15.45:6827/1544784093]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T12:05:14.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:14 smithi165 ceph-mon[49373]: osdmap e106: 8 total, 7 up, 8 in 2024-09-17T12:05:14.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:14 smithi165 ceph-mon[49373]: from='osd.3 [v2:172.21.15.45:6826/1544784093,v1:172.21.15.45:6827/1544784093]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi045", "root=default"]} : dispatch 2024-09-17T12:05:14.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:14 smithi165 ceph-mon[49373]: from='osd.3 [v2:172.21.15.45:6826/1544784093,v1:172.21.15.45:6827/1544784093]' entity='osd.3' 2024-09-17T12:05:14.770 INFO:teuthology.orchestra.run.smithi045.stdout: 5 208901 9.74 MB/sec execute 435 sec latency 53.996 ms 2024-09-17T12:05:15.675 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:15 smithi045 ceph-mon[63880]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T12:05:15.675 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:15 smithi045 ceph-mon[63880]: osd.3 [v2:172.21.15.45:6826/1544784093,v1:172.21.15.45:6827/1544784093] boot 2024-09-17T12:05:15.675 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:15 smithi045 ceph-mon[63880]: osdmap e107: 8 total, 8 up, 8 in 2024-09-17T12:05:15.675 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:15 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-17T12:05:15.675 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:15 smithi045 ceph-mon[63880]: pgmap v170: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 661 op/s; 205/1638 objects degraded (12.515%) 2024-09-17T12:05:15.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:15 smithi165 ceph-mon[49373]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T12:05:15.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:15 smithi165 ceph-mon[49373]: osd.3 [v2:172.21.15.45:6826/1544784093,v1:172.21.15.45:6827/1544784093] boot 2024-09-17T12:05:15.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:15 smithi165 ceph-mon[49373]: osdmap e107: 8 total, 8 up, 8 in 2024-09-17T12:05:15.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:15 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-17T12:05:15.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:15 smithi165 ceph-mon[49373]: pgmap v170: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 661 op/s; 205/1638 objects degraded (12.515%) 2024-09-17T12:05:15.770 INFO:teuthology.orchestra.run.smithi045.stdout: 5 209116 9.72 MB/sec execute 436 sec latency 846.480 ms 2024-09-17T12:05:16.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 209313 9.71 MB/sec execute 437 sec latency 1756.065 ms 2024-09-17T12:05:16.992 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:16 smithi045 ceph-mon[63880]: osdmap e108: 8 total, 8 up, 8 in 2024-09-17T12:05:17.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:16 smithi165 ceph-mon[49373]: osdmap e108: 8 total, 8 up, 8 in 2024-09-17T12:05:17.770 INFO:teuthology.orchestra.run.smithi045.stdout: 5 209530 9.70 MB/sec execute 438 sec latency 1871.487 ms 2024-09-17T12:05:18.006 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:17 smithi045 ceph-mon[63880]: osdmap e109: 8 total, 8 up, 8 in 2024-09-17T12:05:18.006 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:17 smithi045 ceph-mon[63880]: pgmap v173: 97 pgs: 10 active+undersized, 27 active+undersized+degraded, 60 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 1.8 MiB/s wr, 91 op/s; 205/1647 objects degraded (12.447%) 2024-09-17T12:05:18.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:17 smithi165 ceph-mon[49373]: osdmap e109: 8 total, 8 up, 8 in 2024-09-17T12:05:18.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:17 smithi165 ceph-mon[49373]: pgmap v173: 97 pgs: 10 active+undersized, 27 active+undersized+degraded, 60 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 1.8 MiB/s wr, 91 op/s; 205/1647 objects degraded (12.447%) 2024-09-17T12:05:18.770 INFO:teuthology.orchestra.run.smithi045.stdout: 5 209624 9.68 MB/sec execute 439 sec latency 961.943 ms 2024-09-17T12:05:19.058 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:18 smithi045 ceph-mon[63880]: osdmap e110: 8 total, 8 up, 8 in 2024-09-17T12:05:19.058 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:18 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 205/1647 objects degraded (12.447%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T12:05:19.058 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:18 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:19.058 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:18 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:19.058 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:18 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:05:19.058 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:18 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:05:19.059 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:18 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:19.059 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:18 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:19.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:18 smithi165 ceph-mon[49373]: osdmap e110: 8 total, 8 up, 8 in 2024-09-17T12:05:19.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:18 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 205/1647 objects degraded (12.447%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T12:05:19.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:18 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:19.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:18 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:19.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:18 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:05:19.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:18 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:05:19.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:18 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:19.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:18 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:19.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 209896 9.68 MB/sec execute 440 sec latency 1029.515 ms 2024-09-17T12:05:19.846 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T12:05:20.075 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:19 smithi045 ceph-mon[63880]: osdmap e111: 8 total, 8 up, 8 in 2024-09-17T12:05:20.075 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:19 smithi045 ceph-mon[63880]: pgmap v176: 97 pgs: 5 peering, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+undersized, 2 active+undersized+degraded, 81 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 4.7 MiB/s wr, 518 op/s; 186/1608 objects degraded (11.567%); 1.4 MiB/s, 27 objects/s recovering 2024-09-17T12:05:20.075 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:19 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:05:20.075 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:19 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:05:20.075 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:19 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:05:20.075 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:19 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:05:20.076 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:19 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-17T12:05:20.076 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:19 smithi045 ceph-mon[63880]: osdmap e112: 8 total, 8 up, 8 in 2024-09-17T12:05:20.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:19 smithi165 ceph-mon[49373]: osdmap e111: 8 total, 8 up, 8 in 2024-09-17T12:05:20.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:19 smithi165 ceph-mon[49373]: pgmap v176: 97 pgs: 5 peering, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+undersized, 2 active+undersized+degraded, 81 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 4.7 MiB/s wr, 518 op/s; 186/1608 objects degraded (11.567%); 1.4 MiB/s, 27 objects/s recovering 2024-09-17T12:05:20.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:19 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:05:20.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:19 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:05:20.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:19 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:05:20.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:19 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:05:20.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:19 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-17T12:05:20.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:19 smithi165 ceph-mon[49373]: osdmap e112: 8 total, 8 up, 8 in 2024-09-17T12:05:20.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 210269 9.68 MB/sec execute 441 sec latency 819.749 ms 2024-09-17T12:05:20.834 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:20 smithi045 ceph-mon[63880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T12:05:20.834 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:20 smithi045 ceph-mon[63880]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-09-17T12:05:20.834 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:20 smithi045 ceph-mon[63880]: from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:05:21.046 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T12:05:21.046 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (6m) 9s ago 15m 20.4M - 0.27.0 11f11916f8cd 2a2a54402d2a 2024-09-17T12:05:21.046 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (3m) 9s ago 15m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 e545cbf7736d 2024-09-17T12:05:21.046 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (3m) 3m ago 15m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 c1e89e114e4d 2024-09-17T12:05:21.046 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (6m) 9s ago 15m 57.0M - dad864ee21e9 371aaf55379f 2024-09-17T12:05:21.047 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (9m) 9s ago 9m 37.9M - 17.2.0 e1d6a67b021e 6a2a4a998dd1 2024-09-17T12:05:21.047 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (9m) 3m ago 9m 16.0M - 17.2.0 e1d6a67b021e d4c2987b9bac 2024-09-17T12:05:21.047 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:8443,9283,8765 running (5m) 9s ago 17m 520M - 19.3.0-4937-g74984932 6c1fcab85602 ae2f507bd67d 2024-09-17T12:05:21.047 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283,8765 running (4m) 3m ago 15m 453M - 19.3.0-4937-g74984932 6c1fcab85602 53403866652e 2024-09-17T12:05:21.047 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (4m) 9s ago 17m 52.8M 2048M 19.3.0-4937-g74984932 6c1fcab85602 737eec78e0f0 2024-09-17T12:05:21.047 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (4m) 3m ago 15m 36.1M 2048M 19.3.0-4937-g74984932 6c1fcab85602 249eedb4123e 2024-09-17T12:05:21.047 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (9m) 9s ago 9m 279M - 4.0 e1d6a67b021e b8fc749ebb71 2024-09-17T12:05:21.047 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (9m) 3m ago 9m 83.8M - 4.0 e1d6a67b021e 771b27764432 2024-09-17T12:05:21.047 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (6m) 9s ago 15m 11.1M - 1.7.0 72c9c2088986 989b5738cb69 2024-09-17T12:05:21.047 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (5m) 3m ago 15m 11.9M - 1.7.0 72c9c2088986 e1e8ffab3f21 2024-09-17T12:05:21.047 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (2m) 9s ago 14m 338M 1937M 19.3.0-4937-g74984932 6c1fcab85602 20dafd01bd12 2024-09-17T12:05:21.047 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (115s) 9s ago 13m 264M 1937M 19.3.0-4937-g74984932 6c1fcab85602 d39be5014418 2024-09-17T12:05:21.047 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (66s) 9s ago 13m 187M 1937M 19.3.0-4937-g74984932 6c1fcab85602 e13c2ecd369a 2024-09-17T12:05:21.048 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (18s) 9s ago 13m 12.1M 1937M 19.3.0-4937-g74984932 6c1fcab85602 ffd68312d242 2024-09-17T12:05:21.048 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (12m) 3m ago 12m 494M 2704M 17.2.0 e1d6a67b021e e29eb74df25a 2024-09-17T12:05:21.048 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (12m) 3m ago 12m 544M 2704M 17.2.0 e1d6a67b021e 5d900eb9165b 2024-09-17T12:05:21.048 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (11m) 3m ago 11m 466M 2704M 17.2.0 e1d6a67b021e fae3e1bae466 2024-09-17T12:05:21.048 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (11m) 3m ago 11m 371M 2704M 17.2.0 e1d6a67b021e 71b9f2cdb810 2024-09-17T12:05:21.048 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (5m) 9s ago 14m 56.0M - 2.51.0 1d3b7f56885b 61b064ad6609 2024-09-17T12:05:21.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:20 smithi165 ceph-mon[49373]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T12:05:21.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:20 smithi165 ceph-mon[49373]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-09-17T12:05:21.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:20 smithi165 ceph-mon[49373]: from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:05:21.748 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:05:21.748 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T12:05:21.748 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:05:21.748 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:05:21.748 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T12:05:21.748 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:05:21.748 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:05:21.748 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T12:05:21.748 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-17T12:05:21.749 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 4 2024-09-17T12:05:21.749 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:05:21.749 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T12:05:21.749 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T12:05:21.749 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:05:21.749 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T12:05:21.749 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-17T12:05:21.749 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 8 2024-09-17T12:05:21.749 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T12:05:21.749 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:05:21.770 INFO:teuthology.orchestra.run.smithi045.stdout: 5 210659 9.67 MB/sec execute 442 sec latency 42.388 ms 2024-09-17T12:05:21.847 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:21 smithi045 ceph-mon[63880]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:05:21.847 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:21 smithi045 ceph-mon[63880]: pgmap v178: 97 pgs: 5 peering, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 84 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.6 MiB/s wr, 506 op/s; 175/1605 objects degraded (10.903%); 1.4 MiB/s, 27 objects/s recovering 2024-09-17T12:05:21.847 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:21 smithi045 ceph-mon[63880]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:05:21.847 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:21 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:21.847 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:21 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:05:21.847 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:21 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/766554971' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:05:22.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:21 smithi165 ceph-mon[49373]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:05:22.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:21 smithi165 ceph-mon[49373]: pgmap v178: 97 pgs: 5 peering, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 84 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.6 MiB/s wr, 506 op/s; 175/1605 objects degraded (10.903%); 1.4 MiB/s, 27 objects/s recovering 2024-09-17T12:05:22.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:21 smithi165 ceph-mon[49373]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:05:22.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:21 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:22.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:21 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:05:22.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:21 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/766554971' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:05:22.373 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:05:22.373 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T12:05:22.373 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T12:05:22.373 INFO:teuthology.orchestra.run.smithi045.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T12:05:22.373 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [ 2024-09-17T12:05:22.373 INFO:teuthology.orchestra.run.smithi045.stdout: "crash", 2024-09-17T12:05:22.373 INFO:teuthology.orchestra.run.smithi045.stdout: "mon", 2024-09-17T12:05:22.374 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr" 2024-09-17T12:05:22.374 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-17T12:05:22.374 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "10/23 daemons upgraded", 2024-09-17T12:05:22.374 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T12:05:22.374 INFO:teuthology.orchestra.run.smithi045.stdout: "is_paused": false 2024-09-17T12:05:22.374 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:05:22.770 INFO:teuthology.orchestra.run.smithi045.stdout: 5 210937 9.67 MB/sec execute 443 sec latency 79.025 ms 2024-09-17T12:05:23.018 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_WARN Degraded data redundancy: 95/1545 objects degraded (6.149%), 4 pgs degraded 2024-09-17T12:05:23.018 INFO:teuthology.orchestra.run.smithi045.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 95/1545 objects degraded (6.149%), 4 pgs degraded 2024-09-17T12:05:23.018 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.1 is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2024-09-17T12:05:23.018 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.2 is active+recovery_wait+undersized+degraded+remapped, acting [5,6] 2024-09-17T12:05:23.018 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.7 is active+recovery_wait+undersized+degraded+remapped, acting [0,7] 2024-09-17T12:05:23.018 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.b is active+recovery_wait+undersized+degraded+remapped, acting [0,4] 2024-09-17T12:05:23.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 211356 9.69 MB/sec execute 444 sec latency 37.218 ms 2024-09-17T12:05:24.006 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:23 smithi045 ceph-mon[63880]: from='client.34350 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:05:24.007 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:23 smithi045 ceph-mon[63880]: pgmap v179: 97 pgs: 5 peering, 4 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.1 MiB/s wr, 612 op/s; 95/1545 objects degraded (6.149%); 1004 KiB/s, 31 objects/s recovering 2024-09-17T12:05:24.007 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:23 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/2587391453' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:05:24.007 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:23 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 95/1545 objects degraded (6.149%), 4 pgs degraded (PG_DEGRADED) 2024-09-17T12:05:24.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:23 smithi165 ceph-mon[49373]: from='client.34350 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:05:24.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:23 smithi165 ceph-mon[49373]: pgmap v179: 97 pgs: 5 peering, 4 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.1 MiB/s wr, 612 op/s; 95/1545 objects degraded (6.149%); 1004 KiB/s, 31 objects/s recovering 2024-09-17T12:05:24.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:23 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/2587391453' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:05:24.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:23 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 95/1545 objects degraded (6.149%), 4 pgs degraded (PG_DEGRADED) 2024-09-17T12:05:24.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 211785 9.70 MB/sec execute 445 sec latency 43.419 ms 2024-09-17T12:05:25.143 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:25 smithi045 ceph-mon[63880]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 95/1545 objects degraded (6.149%), 4 pgs degraded) 2024-09-17T12:05:25.143 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:25 smithi045 ceph-mon[63880]: Cluster is now healthy 2024-09-17T12:05:25.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:25 smithi165 ceph-mon[49373]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 95/1545 objects degraded (6.149%), 4 pgs degraded) 2024-09-17T12:05:25.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:25 smithi165 ceph-mon[49373]: Cluster is now healthy 2024-09-17T12:05:25.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 212102 9.70 MB/sec execute 446 sec latency 59.222 ms 2024-09-17T12:05:26.164 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:26 smithi045 ceph-mon[63880]: pgmap v180: 97 pgs: 97 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 3.2 MiB/s wr, 314 op/s; 858 KiB/s, 40 objects/s recovering 2024-09-17T12:05:26.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:26 smithi165 ceph-mon[49373]: pgmap v180: 97 pgs: 97 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 3.2 MiB/s wr, 314 op/s; 858 KiB/s, 40 objects/s recovering 2024-09-17T12:05:26.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 212377 9.68 MB/sec execute 447 sec latency 45.300 ms 2024-09-17T12:05:27.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 212701 9.68 MB/sec execute 448 sec latency 62.463 ms 2024-09-17T12:05:28.260 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:28 smithi045 ceph-mon[63880]: pgmap v181: 97 pgs: 97 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 2.7 MiB/s wr, 265 op/s; 535 KiB/s, 24 objects/s recovering 2024-09-17T12:05:28.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:28 smithi165 ceph-mon[49373]: pgmap v181: 97 pgs: 97 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 2.7 MiB/s wr, 265 op/s; 535 KiB/s, 24 objects/s recovering 2024-09-17T12:05:28.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 212997 9.67 MB/sec execute 449 sec latency 68.805 ms 2024-09-17T12:05:29.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 213278 9.67 MB/sec execute 450 sec latency 77.605 ms 2024-09-17T12:05:30.287 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:30 smithi045 ceph-mon[63880]: pgmap v182: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.9 MiB/s wr, 457 op/s; 435 KiB/s, 20 objects/s recovering 2024-09-17T12:05:30.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:30 smithi165 ceph-mon[49373]: pgmap v182: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.9 MiB/s wr, 457 op/s; 435 KiB/s, 20 objects/s recovering 2024-09-17T12:05:30.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 213586 9.67 MB/sec execute 451 sec latency 92.296 ms 2024-09-17T12:05:31.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 213980 9.66 MB/sec execute 452 sec latency 72.201 ms 2024-09-17T12:05:32.073 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:32 smithi045 ceph-mon[63880]: pgmap v183: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.4 MiB/s wr, 411 op/s; 380 KiB/s, 17 objects/s recovering 2024-09-17T12:05:32.073 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:32 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:32.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:32 smithi165 ceph-mon[49373]: pgmap v183: 97 pgs: 97 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.4 MiB/s wr, 411 op/s; 380 KiB/s, 17 objects/s recovering 2024-09-17T12:05:32.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:32 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:32.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 214310 9.65 MB/sec execute 453 sec latency 67.190 ms 2024-09-17T12:05:33.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 214643 9.65 MB/sec execute 454 sec latency 58.478 ms 2024-09-17T12:05:34.165 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:34 smithi045 ceph-mon[63880]: pgmap v184: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.2 MiB/s wr, 545 op/s; 352 KiB/s, 16 objects/s recovering 2024-09-17T12:05:34.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:34 smithi165 ceph-mon[49373]: pgmap v184: 97 pgs: 97 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.2 MiB/s wr, 545 op/s; 352 KiB/s, 16 objects/s recovering 2024-09-17T12:05:34.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 214934 9.67 MB/sec execute 455 sec latency 89.622 ms 2024-09-17T12:05:35.155 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:35 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-17T12:05:35.228 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:35 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-17T12:05:35.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 215262 9.67 MB/sec execute 456 sec latency 60.539 ms 2024-09-17T12:05:36.248 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:36 smithi045 ceph-mon[63880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T12:05:36.248 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:36 smithi045 ceph-mon[63880]: Upgrade: osd.4 is safe to restart 2024-09-17T12:05:36.249 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:36 smithi045 ceph-mon[63880]: pgmap v185: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.6 MiB/s wr, 435 op/s; 352 KiB/s, 9 objects/s recovering 2024-09-17T12:05:36.249 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:36.249 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-17T12:05:36.249 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:05:36.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:36 smithi165 ceph-mon[49373]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T12:05:36.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:36 smithi165 ceph-mon[49373]: Upgrade: osd.4 is safe to restart 2024-09-17T12:05:36.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:36 smithi165 ceph-mon[49373]: pgmap v185: 97 pgs: 97 active+clean; 337 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.6 MiB/s wr, 435 op/s; 352 KiB/s, 9 objects/s recovering 2024-09-17T12:05:36.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:36.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-17T12:05:36.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:05:36.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 215546 9.67 MB/sec execute 457 sec latency 119.457 ms 2024-09-17T12:05:37.295 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:37 smithi045 ceph-mon[63880]: Upgrade: Updating osd.4 2024-09-17T12:05:37.295 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:37 smithi045 ceph-mon[63880]: Deploying daemon osd.4 on smithi165 2024-09-17T12:05:37.295 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:37 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:37.295 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:37 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:05:37.442 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:37 smithi165 ceph-mon[49373]: Upgrade: Updating osd.4 2024-09-17T12:05:37.442 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:37 smithi165 ceph-mon[49373]: Deploying daemon osd.4 on smithi165 2024-09-17T12:05:37.442 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:37 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:37.442 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:37 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:05:37.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 215730 9.65 MB/sec execute 458 sec latency 56.544 ms 2024-09-17T12:05:38.060 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:38 smithi045 ceph-mon[63880]: pgmap v186: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 411 op/s 2024-09-17T12:05:38.391 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:38 smithi165 ceph-mon[49373]: pgmap v186: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 411 op/s 2024-09-17T12:05:38.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 215909 9.64 MB/sec execute 459 sec latency 53.210 ms 2024-09-17T12:05:39.512 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:39 smithi165 systemd[1]: Stopping Ceph osd.4 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:05:39.765 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:39 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4[29932]: 2024-09-17T12:05:39.509+0000 7fdd87b3c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T12:05:39.766 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:39 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4[29932]: 2024-09-17T12:05:39.509+0000 7fdd87b3c700 -1 osd.4 112 *** Got signal Terminated *** 2024-09-17T12:05:39.766 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:39 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4[29932]: 2024-09-17T12:05:39.509+0000 7fdd87b3c700 -1 osd.4 112 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T12:05:39.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 216134 9.63 MB/sec execute 460 sec latency 86.203 ms 2024-09-17T12:05:40.143 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:40 smithi045 ceph-mon[63880]: pgmap v187: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.8 MiB/s wr, 567 op/s 2024-09-17T12:05:40.143 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:40 smithi045 ceph-mon[63880]: osd.4 marked itself down and dead 2024-09-17T12:05:40.467 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:40 smithi165 ceph-mon[49373]: pgmap v187: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.8 MiB/s wr, 567 op/s 2024-09-17T12:05:40.467 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:40 smithi165 ceph-mon[49373]: osd.4 marked itself down and dead 2024-09-17T12:05:40.765 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:40 smithi165 podman[52772]: 2024-09-17 12:05:40.466406228 +0000 UTC m=+1.110688267 container died e29eb74df25a0c3c38762d8e8db891d1f73cbe0ffa52316469cfb46e6b73da27 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., vcs-type=git, release=754, maintainer=Guillaume Abrioux , version=8, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, ceph=True, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64) 2024-09-17T12:05:40.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 216250 9.61 MB/sec execute 461 sec latency 681.311 ms 2024-09-17T12:05:41.067 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:40 smithi165 podman[52772]: 2024-09-17 12:05:40.839718926 +0000 UTC m=+1.484000959 container cleanup e29eb74df25a0c3c38762d8e8db891d1f73cbe0ffa52316469cfb46e6b73da27 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4, maintainer=Guillaume Abrioux , name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, version=8, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870) 2024-09-17T12:05:41.067 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:40 smithi165 bash[52772]: ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4 2024-09-17T12:05:41.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:41 smithi045 ceph-mon[63880]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T12:05:41.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:41 smithi045 ceph-mon[63880]: osdmap e113: 8 total, 7 up, 8 in 2024-09-17T12:05:41.446 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:41 smithi165 ceph-mon[49373]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T12:05:41.446 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:41 smithi165 ceph-mon[49373]: osdmap e113: 8 total, 7 up, 8 in 2024-09-17T12:05:41.446 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:41 smithi165 podman[52787]: 2024-09-17 12:05:41.07436965 +0000 UTC m=+0.604785164 container remove e29eb74df25a0c3c38762d8e8db891d1f73cbe0ffa52316469cfb46e6b73da27 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.component=centos-stream-container, release=754, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux ) 2024-09-17T12:05:41.765 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:41 smithi165 podman[52876]: 2024-09-17 12:05:41.348371247 +0000 UTC m=+0.016128767 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:05:41.765 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:41 smithi165 podman[52876]: 2024-09-17 12:05:41.451866806 +0000 UTC m=+0.119624321 container create 1fe93968486baffc66b2d77ae93906dec808cfff3934578826717d11e3899433 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=main-7498493, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-17T12:05:41.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 216403 9.60 MB/sec execute 462 sec latency 1027.545 ms 2024-09-17T12:05:42.049 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:41 smithi165 podman[52876]: 2024-09-17 12:05:41.939409851 +0000 UTC m=+0.607167373 container init 1fe93968486baffc66b2d77ae93906dec808cfff3934578826717d11e3899433 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=main-7498493) 2024-09-17T12:05:42.049 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:41 smithi165 podman[52876]: 2024-09-17 12:05:41.949890586 +0000 UTC m=+0.617648099 container start 1fe93968486baffc66b2d77ae93906dec808cfff3934578826717d11e3899433 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=main-7498493, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T12:05:42.049 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:42 smithi165 podman[52876]: 2024-09-17 12:05:42.047912312 +0000 UTC m=+0.715669828 container attach 1fe93968486baffc66b2d77ae93906dec808cfff3934578826717d11e3899433 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=main-7498493) 2024-09-17T12:05:42.123 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:42 smithi045 ceph-mon[63880]: pgmap v189: 97 pgs: 14 stale+active+clean, 83 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 439 op/s 2024-09-17T12:05:42.123 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:42 smithi045 ceph-mon[63880]: osdmap e114: 8 total, 7 up, 8 in 2024-09-17T12:05:42.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:42 smithi165 ceph-mon[49373]: pgmap v189: 97 pgs: 14 stale+active+clean, 83 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 439 op/s 2024-09-17T12:05:42.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:42 smithi165 ceph-mon[49373]: osdmap e114: 8 total, 7 up, 8 in 2024-09-17T12:05:42.516 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:42 smithi165 podman[52876]: 2024-09-17 12:05:42.168633907 +0000 UTC m=+0.836391421 container died 1fe93968486baffc66b2d77ae93906dec808cfff3934578826717d11e3899433 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=main-7498493, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-17T12:05:42.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 216608 9.59 MB/sec execute 463 sec latency 64.170 ms 2024-09-17T12:05:43.144 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:42 smithi165 podman[52899]: 2024-09-17 12:05:42.994781661 +0000 UTC m=+0.814987025 container remove 1fe93968486baffc66b2d77ae93906dec808cfff3934578826717d11e3899433 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True) 2024-09-17T12:05:43.144 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:43 smithi165 systemd[1]: ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-17T12:05:43.144 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:43 smithi165 systemd[1]: Stopped Ceph osd.4 for a12537de-74ea-11ef-bceb-c7b262605968. 2024-09-17T12:05:43.145 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:43 smithi165 systemd[1]: ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.4.service: Consumed 3min 36.842s CPU time. 2024-09-17T12:05:43.177 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:43 smithi045 ceph-mon[63880]: Health check failed: Degraded data redundancy: 139/1521 objects degraded (9.139%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T12:05:43.455 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:43 smithi165 ceph-mon[49373]: Health check failed: Degraded data redundancy: 139/1521 objects degraded (9.139%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T12:05:43.455 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:43 smithi165 systemd[1]: Starting Ceph osd.4 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:05:43.765 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:43 smithi165 podman[53012]: 2024-09-17 12:05:43.357005286 +0000 UTC m=+0.017043644 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:05:43.765 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:43 smithi165 podman[53012]: 2024-09-17 12:05:43.462870876 +0000 UTC m=+0.122909221 container create fd6860176f2430e2a4ca98af52cc4bc4c984514a63cb540ff45a415d2e51b7c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=main-7498493, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True) 2024-09-17T12:05:43.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 216922 9.59 MB/sec execute 464 sec latency 50.898 ms 2024-09-17T12:05:44.146 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:43 smithi165 podman[53012]: 2024-09-17 12:05:43.816959 +0000 UTC m=+0.476997369 container init fd6860176f2430e2a4ca98af52cc4bc4c984514a63cb540ff45a415d2e51b7c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-activate, RELEASE=main-7498493, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-17T12:05:44.147 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:43 smithi165 podman[53012]: 2024-09-17 12:05:43.821449936 +0000 UTC m=+0.481488281 container start fd6860176f2430e2a4ca98af52cc4bc4c984514a63cb540ff45a415d2e51b7c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-17T12:05:44.147 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:43 smithi165 podman[53012]: 2024-09-17 12:05:43.925255492 +0000 UTC m=+0.585293842 container attach fd6860176f2430e2a4ca98af52cc4bc4c984514a63cb540ff45a415d2e51b7c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=) 2024-09-17T12:05:44.147 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:43 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-activate[53027]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:05:44.147 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:43 smithi165 bash[53012]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:05:44.147 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:43 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-activate[53027]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:05:44.147 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:43 smithi165 bash[53012]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:05:44.230 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:44 smithi045 ceph-mon[63880]: pgmap v191: 97 pgs: 6 active+undersized, 7 stale+active+clean, 19 active+undersized+degraded, 65 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.2 MiB/s wr, 417 op/s; 139/1521 objects degraded (9.139%) 2024-09-17T12:05:44.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:44 smithi165 ceph-mon[49373]: pgmap v191: 97 pgs: 6 active+undersized, 7 stale+active+clean, 19 active+undersized+degraded, 65 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 4.2 MiB/s wr, 417 op/s; 139/1521 objects degraded (9.139%) 2024-09-17T12:05:44.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 217277 9.58 MB/sec execute 465 sec latency 61.976 ms 2024-09-17T12:05:45.652 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:45 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-activate[53027]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T12:05:45.652 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:45 smithi165 bash[53012]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T12:05:45.652 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:45 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-activate[53027]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T12:05:45.653 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:45 smithi165 bash[53012]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T12:05:45.653 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:45 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-activate[53027]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T12:05:45.653 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:45 smithi165 bash[53012]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T12:05:45.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 217598 9.57 MB/sec execute 466 sec latency 40.714 ms 2024-09-17T12:05:45.981 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:45 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-activate[53027]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T12:05:45.981 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:45 smithi165 bash[53012]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T12:05:45.981 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:45 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-activate[53027]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-17T12:05:45.982 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:45 smithi165 bash[53012]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-17T12:05:45.982 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:45 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-activate[53027]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T12:05:45.982 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:45 smithi165 bash[53012]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T12:05:45.982 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:45 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-activate[53027]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-17T12:05:45.982 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:45 smithi165 bash[53012]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-17T12:05:45.982 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:45 smithi165 podman[53012]: 2024-09-17 12:05:45.680571096 +0000 UTC m=+2.340609443 container died fd6860176f2430e2a4ca98af52cc4bc4c984514a63cb540ff45a415d2e51b7c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-activate, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-7498493, io.buildah.version=1.37.2) 2024-09-17T12:05:46.262 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:46 smithi045 ceph-mon[63880]: pgmap v192: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.6 MiB/s wr, 439 op/s; 225/1443 objects degraded (15.593%) 2024-09-17T12:05:46.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:46 smithi165 ceph-mon[49373]: pgmap v192: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.6 MiB/s wr, 439 op/s; 225/1443 objects degraded (15.593%) 2024-09-17T12:05:46.659 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:46 smithi165 podman[53170]: 2024-09-17 12:05:46.407036735 +0000 UTC m=+0.719638503 container remove fd6860176f2430e2a4ca98af52cc4bc4c984514a63cb540ff45a415d2e51b7c7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-activate, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=main-7498493) 2024-09-17T12:05:46.659 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:46 smithi165 podman[53230]: 2024-09-17 12:05:46.526936358 +0000 UTC m=+0.017267095 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:05:46.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 217867 9.57 MB/sec execute 467 sec latency 72.751 ms 2024-09-17T12:05:46.915 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:46 smithi165 podman[53230]: 2024-09-17 12:05:46.658162404 +0000 UTC m=+0.148493115 container create 589de85559fd665c9f3190da54822cafc42714b0a558935878318242edfab795 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=main-7498493, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T12:05:46.915 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:46 smithi165 podman[53230]: 2024-09-17 12:05:46.888422028 +0000 UTC m=+0.378752752 container init 589de85559fd665c9f3190da54822cafc42714b0a558935878318242edfab795 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=main-7498493) 2024-09-17T12:05:46.916 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:46 smithi165 podman[53230]: 2024-09-17 12:05:46.891659468 +0000 UTC m=+0.381990188 container start 589de85559fd665c9f3190da54822cafc42714b0a558935878318242edfab795 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=main-7498493, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD) 2024-09-17T12:05:47.267 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:46 smithi165 ceph-osd[53249]: -- 172.21.15.165:0/4273702217 <== mon.1 v2:172.21.15.165:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557513d3c000 con 0x557512f49c00 2024-09-17T12:05:47.297 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:46 smithi165 bash[53230]: 589de85559fd665c9f3190da54822cafc42714b0a558935878318242edfab795 2024-09-17T12:05:47.297 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:46 smithi165 systemd[1]: Started Ceph osd.4 for a12537de-74ea-11ef-bceb-c7b262605968. 2024-09-17T12:05:47.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 218131 9.56 MB/sec execute 468 sec latency 63.386 ms 2024-09-17T12:05:48.103 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:48 smithi045 ceph-mon[63880]: pgmap v193: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 1.9 MiB/s wr, 192 op/s; 225/1443 objects degraded (15.593%) 2024-09-17T12:05:48.103 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:48 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:48.103 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:48 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:48.103 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:48 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:05:48.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:48 smithi165 ceph-mon[49373]: pgmap v193: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 1.9 MiB/s wr, 192 op/s; 225/1443 objects degraded (15.593%) 2024-09-17T12:05:48.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:48 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:48.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:48 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:48.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:48 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:05:48.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 218442 9.57 MB/sec execute 469 sec latency 46.812 ms 2024-09-17T12:05:49.056 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:48 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4[53245]: 2024-09-17T12:05:48.776+0000 7f4e9c06a640 -1 Falling back to public interface 2024-09-17T12:05:49.118 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:49 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 225/1443 objects degraded (15.593%), 28 pgs degraded (PG_DEGRADED) 2024-09-17T12:05:49.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:49 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 225/1443 objects degraded (15.593%), 28 pgs degraded (PG_DEGRADED) 2024-09-17T12:05:49.771 INFO:teuthology.orchestra.run.smithi045.stdout: 5 218781 9.58 MB/sec execute 470 sec latency 50.857 ms 2024-09-17T12:05:50.186 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:50 smithi045 ceph-mon[63880]: pgmap v194: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.7 MiB/s wr, 403 op/s; 236/1545 objects degraded (15.275%) 2024-09-17T12:05:50.499 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:50 smithi165 ceph-mon[49373]: pgmap v194: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.7 MiB/s wr, 403 op/s; 236/1545 objects degraded (15.275%) 2024-09-17T12:05:50.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 219069 9.58 MB/sec execute 471 sec latency 86.484 ms 2024-09-17T12:05:51.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 219263 9.57 MB/sec execute 472 sec latency 67.921 ms 2024-09-17T12:05:52.009 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:51 smithi045 ceph-mon[63880]: pgmap v195: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.3 MiB/s wr, 362 op/s; 238/1551 objects degraded (15.345%) 2024-09-17T12:05:52.009 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:52.009 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:52.009 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:52.009 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:05:52.009 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:52.009 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:51 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:52.236 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:51 smithi165 ceph-mon[49373]: pgmap v195: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.3 MiB/s wr, 362 op/s; 238/1551 objects degraded (15.345%) 2024-09-17T12:05:52.237 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:51 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:52.237 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:51 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:52.237 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:51 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:52.237 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:51 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:05:52.237 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:51 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:52.237 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:51 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:52.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 219573 9.56 MB/sec execute 473 sec latency 125.259 ms 2024-09-17T12:05:53.549 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T12:05:53.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 219921 9.55 MB/sec execute 474 sec latency 62.043 ms 2024-09-17T12:05:54.095 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:53 smithi045 ceph-mon[63880]: pgmap v196: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.2 MiB/s wr, 446 op/s; 242/1584 objects degraded (15.278%) 2024-09-17T12:05:54.095 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:53 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 242/1584 objects degraded (15.278%), 28 pgs degraded (PG_DEGRADED) 2024-09-17T12:05:54.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:53 smithi165 ceph-mon[49373]: pgmap v196: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.2 MiB/s wr, 446 op/s; 242/1584 objects degraded (15.278%) 2024-09-17T12:05:54.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:53 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 242/1584 objects degraded (15.278%), 28 pgs degraded (PG_DEGRADED) 2024-09-17T12:05:54.552 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T12:05:54.553 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (7m) 43s ago 16m 20.4M - 0.27.0 11f11916f8cd 2a2a54402d2a 2024-09-17T12:05:54.553 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (3m) 43s ago 16m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 e545cbf7736d 2024-09-17T12:05:54.553 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (3m) 3s ago 15m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 c1e89e114e4d 2024-09-17T12:05:54.553 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (7m) 43s ago 16m 57.0M - dad864ee21e9 371aaf55379f 2024-09-17T12:05:54.553 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (10m) 43s ago 10m 37.9M - 17.2.0 e1d6a67b021e 6a2a4a998dd1 2024-09-17T12:05:54.553 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (10m) 3s ago 10m 16.7M - 17.2.0 e1d6a67b021e d4c2987b9bac 2024-09-17T12:05:54.553 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:8443,9283,8765 running (6m) 43s ago 17m 520M - 19.3.0-4937-g74984932 6c1fcab85602 ae2f507bd67d 2024-09-17T12:05:54.553 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283,8765 running (5m) 3s ago 15m 457M - 19.3.0-4937-g74984932 6c1fcab85602 53403866652e 2024-09-17T12:05:54.553 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (5m) 43s ago 17m 52.8M 2048M 19.3.0-4937-g74984932 6c1fcab85602 737eec78e0f0 2024-09-17T12:05:54.553 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (4m) 3s ago 15m 41.8M 2048M 19.3.0-4937-g74984932 6c1fcab85602 249eedb4123e 2024-09-17T12:05:54.553 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (10m) 43s ago 10m 279M - 4.0 e1d6a67b021e b8fc749ebb71 2024-09-17T12:05:54.553 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (10m) 3s ago 10m 86.4M - 4.0 e1d6a67b021e 771b27764432 2024-09-17T12:05:54.553 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (6m) 43s ago 16m 11.1M - 1.7.0 72c9c2088986 989b5738cb69 2024-09-17T12:05:54.554 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (6m) 3s ago 15m 12.3M - 1.7.0 72c9c2088986 e1e8ffab3f21 2024-09-17T12:05:54.554 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (3m) 43s ago 14m 338M 1937M 19.3.0-4937-g74984932 6c1fcab85602 20dafd01bd12 2024-09-17T12:05:54.554 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (2m) 43s ago 14m 264M 1937M 19.3.0-4937-g74984932 6c1fcab85602 d39be5014418 2024-09-17T12:05:54.554 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (100s) 43s ago 14m 187M 1937M 19.3.0-4937-g74984932 6c1fcab85602 e13c2ecd369a 2024-09-17T12:05:54.554 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (52s) 43s ago 13m 12.1M 1937M 19.3.0-4937-g74984932 6c1fcab85602 ffd68312d242 2024-09-17T12:05:54.554 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (8s) 3s ago 13m 12.1M 2704M 19.3.0-4937-g74984932 6c1fcab85602 589de85559fd 2024-09-17T12:05:54.554 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (12m) 3s ago 12m 508M 2704M 17.2.0 e1d6a67b021e 5d900eb9165b 2024-09-17T12:05:54.554 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (12m) 3s ago 12m 426M 2704M 17.2.0 e1d6a67b021e fae3e1bae466 2024-09-17T12:05:54.554 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (12m) 3s ago 12m 701M 2704M 17.2.0 e1d6a67b021e 71b9f2cdb810 2024-09-17T12:05:54.554 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (5m) 43s ago 15m 56.0M - 2.51.0 1d3b7f56885b 61b064ad6609 2024-09-17T12:05:54.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 220240 9.55 MB/sec execute 475 sec latency 52.473 ms 2024-09-17T12:05:55.104 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:54 smithi045 ceph-mon[63880]: from='client.44267 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:05:55.104 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:54 smithi045 ceph-mon[63880]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:05:55.262 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:05:55.263 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T12:05:55.263 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:05:55.263 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:05:55.263 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T12:05:55.263 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:05:55.263 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:05:55.263 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T12:05:55.263 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-17T12:05:55.263 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 4 2024-09-17T12:05:55.263 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:05:55.263 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T12:05:55.263 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T12:05:55.263 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:05:55.264 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T12:05:55.264 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-17T12:05:55.264 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 8 2024-09-17T12:05:55.264 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T12:05:55.264 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:05:55.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:54 smithi165 ceph-mon[49373]: from='client.44267 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:05:55.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:54 smithi165 ceph-mon[49373]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:05:55.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 220768 9.56 MB/sec execute 476 sec latency 42.758 ms 2024-09-17T12:05:55.814 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:05:55.814 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T12:05:55.814 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T12:05:55.814 INFO:teuthology.orchestra.run.smithi045.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T12:05:55.814 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [ 2024-09-17T12:05:55.815 INFO:teuthology.orchestra.run.smithi045.stdout: "crash", 2024-09-17T12:05:55.815 INFO:teuthology.orchestra.run.smithi045.stdout: "mon", 2024-09-17T12:05:55.815 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr" 2024-09-17T12:05:55.815 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-17T12:05:55.815 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "11/23 daemons upgraded", 2024-09-17T12:05:55.815 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T12:05:55.815 INFO:teuthology.orchestra.run.smithi045.stdout: "is_paused": false 2024-09-17T12:05:55.815 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:05:56.004 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:55 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4[53245]: 2024-09-17T12:05:55.821+0000 7f4e9c06a640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-17T12:05:56.142 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:56 smithi045 ceph-mon[63880]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:05:56.142 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:56 smithi045 ceph-mon[63880]: pgmap v197: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.9 MiB/s wr, 362 op/s; 257/1680 objects degraded (15.298%) 2024-09-17T12:05:56.142 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:56 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/3142000049' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:05:56.142 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:56.142 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:56.143 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:05:56.143 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:05:56.143 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:56.143 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:56.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:56 smithi165 ceph-mon[49373]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:05:56.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:56 smithi165 ceph-mon[49373]: pgmap v197: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.9 MiB/s wr, 362 op/s; 257/1680 objects degraded (15.298%) 2024-09-17T12:05:56.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:56 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/3142000049' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:05:56.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:56.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:56.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:05:56.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:05:56.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:56.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:05:56.521 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 257/1680 objects degraded (15.298%), 28 pgs degraded 2024-09-17T12:05:56.522 INFO:teuthology.orchestra.run.smithi045.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T12:05:56.522 INFO:teuthology.orchestra.run.smithi045.stdout: osd.4 (root=default,host=smithi165) is down 2024-09-17T12:05:56.522 INFO:teuthology.orchestra.run.smithi045.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 257/1680 objects degraded (15.298%), 28 pgs degraded 2024-09-17T12:05:56.522 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.5 is active+undersized+degraded, acting [7,0] 2024-09-17T12:05:56.522 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.6 is active+undersized+degraded, acting [1,6] 2024-09-17T12:05:56.522 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.b is active+undersized+degraded, acting [7,5] 2024-09-17T12:05:56.522 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.d is active+undersized+degraded, acting [1,3] 2024-09-17T12:05:56.522 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.f is active+undersized+degraded, acting [0,7] 2024-09-17T12:05:56.523 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.11 is active+undersized+degraded, acting [6,1] 2024-09-17T12:05:56.523 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.13 is active+undersized+degraded, acting [0,3] 2024-09-17T12:05:56.523 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.18 is active+undersized+degraded, acting [5,7] 2024-09-17T12:05:56.523 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.1a is active+undersized+degraded, acting [6,7] 2024-09-17T12:05:56.523 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.1f is active+undersized+degraded, acting [0,2] 2024-09-17T12:05:56.523 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.1 is active+undersized+degraded, acting [0,3] 2024-09-17T12:05:56.523 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.3 is active+undersized+degraded, acting [0,6] 2024-09-17T12:05:56.523 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.6 is active+undersized+degraded, acting [0,1] 2024-09-17T12:05:56.524 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.9 is active+undersized+degraded, acting [2,7] 2024-09-17T12:05:56.524 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.a is active+undersized+degraded, acting [6,1] 2024-09-17T12:05:56.524 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.b is active+undersized+degraded, acting [3,0] 2024-09-17T12:05:56.524 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.e is active+undersized+degraded, acting [7,1] 2024-09-17T12:05:56.524 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.f is active+undersized+degraded, acting [7,0] 2024-09-17T12:05:56.524 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.11 is active+undersized+degraded, acting [7,6] 2024-09-17T12:05:56.524 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.13 is active+undersized+degraded, acting [7,2] 2024-09-17T12:05:56.524 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.14 is active+undersized+degraded, acting [7,6] 2024-09-17T12:05:56.524 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.15 is active+undersized+degraded, acting [7,3] 2024-09-17T12:05:56.525 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.19 is active+undersized+degraded, acting [1,3] 2024-09-17T12:05:56.525 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.1a is active+undersized+degraded, acting [1,2] 2024-09-17T12:05:56.525 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.1b is active+undersized+degraded, acting [0,7] 2024-09-17T12:05:56.525 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.1c is active+undersized+degraded, acting [5,1] 2024-09-17T12:05:56.525 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.1d is active+undersized+degraded, acting [5,6] 2024-09-17T12:05:56.525 INFO:teuthology.orchestra.run.smithi045.stdout: pg 4.5 is active+undersized+degraded, acting [6,7] 2024-09-17T12:05:56.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 221133 9.55 MB/sec execute 477 sec latency 71.801 ms 2024-09-17T12:05:57.152 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:57 smithi045 ceph-mon[63880]: from='client.34374 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:05:57.152 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:57 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:05:57.152 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:57 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:05:57.152 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:57 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:05:57.152 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:57 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:05:57.152 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:57 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-17T12:05:57.152 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:57 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/2421371162' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:05:57.265 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:57 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4[53245]: 2024-09-17T12:05:57.012+0000 7f4e9c06a640 -1 osd.4 112 log_to_monitors true 2024-09-17T12:05:57.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:57 smithi165 ceph-mon[49373]: from='client.34374 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:05:57.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:57 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:05:57.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:57 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:05:57.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:57 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:05:57.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:57 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:05:57.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:57 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-17T12:05:57.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:57 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/2421371162' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:05:57.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 221415 9.55 MB/sec execute 478 sec latency 53.986 ms 2024-09-17T12:05:58.095 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:58 smithi045 ceph-mon[63880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T12:05:58.095 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:58 smithi045 ceph-mon[63880]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-17T12:05:58.095 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:58 smithi045 ceph-mon[63880]: pgmap v198: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.5 MiB/s wr, 342 op/s; 257/1680 objects degraded (15.298%) 2024-09-17T12:05:58.096 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:58 smithi045 ceph-mon[63880]: from='osd.4 [v2:172.21.15.165:6800/1234049210,v1:172.21.15.165:6801/1234049210]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-17T12:05:58.096 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:58 smithi045 ceph-mon[63880]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-17T12:05:58.314 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:58 smithi165 ceph-mon[49373]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T12:05:58.314 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:58 smithi165 ceph-mon[49373]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-17T12:05:58.315 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:58 smithi165 ceph-mon[49373]: pgmap v198: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.5 MiB/s wr, 342 op/s; 257/1680 objects degraded (15.298%) 2024-09-17T12:05:58.315 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:58 smithi165 ceph-mon[49373]: from='osd.4 [v2:172.21.15.165:6800/1234049210,v1:172.21.15.165:6801/1234049210]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-17T12:05:58.315 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:58 smithi165 ceph-mon[49373]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-17T12:05:58.315 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:05:58 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4[53245]: 2024-09-17T12:05:58.041+0000 7f4e93e14640 -1 osd.4 112 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T12:05:58.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 221737 9.55 MB/sec execute 479 sec latency 69.181 ms 2024-09-17T12:05:59.179 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:59 smithi045 ceph-mon[63880]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T12:05:59.179 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:59 smithi045 ceph-mon[63880]: osdmap e115: 8 total, 7 up, 8 in 2024-09-17T12:05:59.179 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:59 smithi045 ceph-mon[63880]: from='osd.4 [v2:172.21.15.165:6800/1234049210,v1:172.21.15.165:6801/1234049210]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi165", "root=default"]} : dispatch 2024-09-17T12:05:59.179 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:59 smithi045 ceph-mon[63880]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi165", "root=default"]} : dispatch 2024-09-17T12:05:59.179 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:05:59 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 257/1680 objects degraded (15.298%), 28 pgs degraded (PG_DEGRADED) 2024-09-17T12:05:59.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:59 smithi165 ceph-mon[49373]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T12:05:59.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:59 smithi165 ceph-mon[49373]: osdmap e115: 8 total, 7 up, 8 in 2024-09-17T12:05:59.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:59 smithi165 ceph-mon[49373]: from='osd.4 [v2:172.21.15.165:6800/1234049210,v1:172.21.15.165:6801/1234049210]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi165", "root=default"]} : dispatch 2024-09-17T12:05:59.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:59 smithi165 ceph-mon[49373]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi165", "root=default"]} : dispatch 2024-09-17T12:05:59.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:05:59 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 257/1680 objects degraded (15.298%), 28 pgs degraded (PG_DEGRADED) 2024-09-17T12:05:59.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 221892 9.55 MB/sec execute 480 sec latency 711.628 ms 2024-09-17T12:06:00.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:00 smithi045 ceph-mon[63880]: pgmap v200: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.0 MiB/s wr, 479 op/s; 256/1614 objects degraded (15.861%) 2024-09-17T12:06:00.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:00 smithi045 ceph-mon[63880]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T12:06:00.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:00 smithi045 ceph-mon[63880]: osd.4 [v2:172.21.15.165:6800/1234049210,v1:172.21.15.165:6801/1234049210] boot 2024-09-17T12:06:00.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:00 smithi045 ceph-mon[63880]: osdmap e116: 8 total, 8 up, 8 in 2024-09-17T12:06:00.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:00 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-17T12:06:00.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:00 smithi045 ceph-mon[63880]: osdmap e117: 8 total, 8 up, 8 in 2024-09-17T12:06:00.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:00 smithi165 ceph-mon[49373]: pgmap v200: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.0 MiB/s wr, 479 op/s; 256/1614 objects degraded (15.861%) 2024-09-17T12:06:00.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:00 smithi165 ceph-mon[49373]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T12:06:00.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:00 smithi165 ceph-mon[49373]: osd.4 [v2:172.21.15.165:6800/1234049210,v1:172.21.15.165:6801/1234049210] boot 2024-09-17T12:06:00.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:00 smithi165 ceph-mon[49373]: osdmap e116: 8 total, 8 up, 8 in 2024-09-17T12:06:00.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:00 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-17T12:06:00.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:00 smithi165 ceph-mon[49373]: osdmap e117: 8 total, 8 up, 8 in 2024-09-17T12:06:00.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 221900 9.53 MB/sec execute 481 sec latency 1711.676 ms 2024-09-17T12:06:01.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 222180 9.53 MB/sec execute 482 sec latency 2052.276 ms 2024-09-17T12:06:02.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:02 smithi045 ceph-mon[63880]: pgmap v203: 97 pgs: 1 peering, 12 active+undersized, 28 active+undersized+degraded, 56 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 461 op/s; 256/1611 objects degraded (15.891%) 2024-09-17T12:06:02.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:02 smithi045 ceph-mon[63880]: osdmap e118: 8 total, 8 up, 8 in 2024-09-17T12:06:02.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:02 smithi165 ceph-mon[49373]: pgmap v203: 97 pgs: 1 peering, 12 active+undersized, 28 active+undersized+degraded, 56 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.2 MiB/s wr, 461 op/s; 256/1611 objects degraded (15.891%) 2024-09-17T12:06:02.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:02 smithi165 ceph-mon[49373]: osdmap e118: 8 total, 8 up, 8 in 2024-09-17T12:06:02.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 222264 9.51 MB/sec execute 483 sec latency 715.858 ms 2024-09-17T12:06:03.303 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:03 smithi045 ceph-mon[63880]: osdmap e119: 8 total, 8 up, 8 in 2024-09-17T12:06:03.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:03 smithi165 ceph-mon[49373]: osdmap e119: 8 total, 8 up, 8 in 2024-09-17T12:06:03.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 222533 9.52 MB/sec execute 484 sec latency 1040.019 ms 2024-09-17T12:06:04.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:04 smithi045 ceph-mon[63880]: pgmap v206: 97 pgs: 11 active+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized, 9 active+undersized+degraded, 64 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.7 MiB/s wr, 308 op/s; 105/1596 objects degraded (6.579%); 4.1 MiB/s, 56 keys/s, 56 objects/s recovering 2024-09-17T12:06:04.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:04 smithi045 ceph-mon[63880]: osdmap e120: 8 total, 8 up, 8 in 2024-09-17T12:06:04.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:04 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 105/1596 objects degraded (6.579%), 10 pgs degraded (PG_DEGRADED) 2024-09-17T12:06:04.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:04 smithi165 ceph-mon[49373]: pgmap v206: 97 pgs: 11 active+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized, 9 active+undersized+degraded, 64 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.7 MiB/s wr, 308 op/s; 105/1596 objects degraded (6.579%); 4.1 MiB/s, 56 keys/s, 56 objects/s recovering 2024-09-17T12:06:04.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:04 smithi165 ceph-mon[49373]: osdmap e120: 8 total, 8 up, 8 in 2024-09-17T12:06:04.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:04 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 105/1596 objects degraded (6.579%), 10 pgs degraded (PG_DEGRADED) 2024-09-17T12:06:04.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 222743 9.51 MB/sec execute 485 sec latency 53.575 ms 2024-09-17T12:06:05.100 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:05 smithi045 ceph-mon[63880]: osdmap e121: 8 total, 8 up, 8 in 2024-09-17T12:06:05.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:05 smithi165 ceph-mon[49373]: osdmap e121: 8 total, 8 up, 8 in 2024-09-17T12:06:05.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 222952 9.49 MB/sec execute 486 sec latency 75.208 ms 2024-09-17T12:06:06.124 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:06 smithi045 ceph-mon[63880]: pgmap v209: 97 pgs: 10 active+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.2 MiB/s wr, 363 op/s; 9/1596 objects degraded (0.564%); 4.5 MiB/s, 56 keys/s, 85 objects/s recovering 2024-09-17T12:06:06.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:06 smithi165 ceph-mon[49373]: pgmap v209: 97 pgs: 10 active+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.2 MiB/s wr, 363 op/s; 9/1596 objects degraded (0.564%); 4.5 MiB/s, 56 keys/s, 85 objects/s recovering 2024-09-17T12:06:06.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 223123 9.48 MB/sec execute 487 sec latency 71.026 ms 2024-09-17T12:06:07.661 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:07 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:07.661 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:07 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:06:07.661 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:07 smithi045 ceph-mon[63880]: pgmap v210: 97 pgs: 10 active+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.8 MiB/s wr, 246 op/s; 9/1596 objects degraded (0.564%); 3.1 MiB/s, 38 keys/s, 57 objects/s recovering 2024-09-17T12:06:07.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:07 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:07.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:07 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:06:07.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:07 smithi165 ceph-mon[49373]: pgmap v210: 97 pgs: 10 active+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 85 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.8 MiB/s wr, 246 op/s; 9/1596 objects degraded (0.564%); 3.1 MiB/s, 38 keys/s, 57 objects/s recovering 2024-09-17T12:06:07.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 223362 9.47 MB/sec execute 488 sec latency 84.656 ms 2024-09-17T12:06:08.464 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:08 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 9/1596 objects degraded (0.564%), 1 pg degraded (PG_DEGRADED) 2024-09-17T12:06:08.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:08 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 9/1596 objects degraded (0.564%), 1 pg degraded (PG_DEGRADED) 2024-09-17T12:06:08.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 223588 9.46 MB/sec execute 489 sec latency 59.369 ms 2024-09-17T12:06:09.503 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:09 smithi045 ceph-mon[63880]: pgmap v211: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 3.0 MiB/s wr, 269 op/s; 379 KiB/s, 19 objects/s recovering 2024-09-17T12:06:09.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:09 smithi165 ceph-mon[49373]: pgmap v211: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 3.0 MiB/s wr, 269 op/s; 379 KiB/s, 19 objects/s recovering 2024-09-17T12:06:09.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 223927 9.47 MB/sec execute 490 sec latency 44.302 ms 2024-09-17T12:06:10.470 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:10 smithi045 ceph-mon[63880]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9/1596 objects degraded (0.564%), 1 pg degraded) 2024-09-17T12:06:10.470 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:10 smithi045 ceph-mon[63880]: Cluster is now healthy 2024-09-17T12:06:10.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:10 smithi165 ceph-mon[49373]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9/1596 objects degraded (0.564%), 1 pg degraded) 2024-09-17T12:06:10.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:10 smithi165 ceph-mon[49373]: Cluster is now healthy 2024-09-17T12:06:10.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 224276 9.46 MB/sec execute 491 sec latency 45.494 ms 2024-09-17T12:06:11.494 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:11 smithi045 ceph-mon[63880]: pgmap v212: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 2.6 MiB/s wr, 234 op/s; 326 KiB/s, 16 objects/s recovering 2024-09-17T12:06:11.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:11 smithi165 ceph-mon[49373]: pgmap v212: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 2.6 MiB/s wr, 234 op/s; 326 KiB/s, 16 objects/s recovering 2024-09-17T12:06:11.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 224668 9.45 MB/sec execute 492 sec latency 50.623 ms 2024-09-17T12:06:12.515 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:12 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-17T12:06:12.515 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:12 smithi045 ceph-mon[63880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T12:06:12.515 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:12 smithi045 ceph-mon[63880]: Upgrade: osd.5 is safe to restart 2024-09-17T12:06:12.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:12 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-17T12:06:12.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:12 smithi165 ceph-mon[49373]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T12:06:12.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:12 smithi165 ceph-mon[49373]: Upgrade: osd.5 is safe to restart 2024-09-17T12:06:12.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 224940 9.45 MB/sec execute 493 sec latency 51.191 ms 2024-09-17T12:06:13.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 225205 9.45 MB/sec execute 494 sec latency 79.942 ms 2024-09-17T12:06:14.048 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:14 smithi045 ceph-mon[63880]: pgmap v213: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 2.9 MiB/s wr, 258 op/s; 128 KiB/s, 10 objects/s recovering 2024-09-17T12:06:14.048 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:14 smithi045 ceph-mon[63880]: Upgrade: Updating osd.5 2024-09-17T12:06:14.048 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:14 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:14.048 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:14 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-17T12:06:14.049 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:14 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:06:14.049 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:14 smithi045 ceph-mon[63880]: Deploying daemon osd.5 on smithi165 2024-09-17T12:06:14.382 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:14 smithi165 ceph-mon[49373]: pgmap v213: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 2.9 MiB/s wr, 258 op/s; 128 KiB/s, 10 objects/s recovering 2024-09-17T12:06:14.382 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:14 smithi165 ceph-mon[49373]: Upgrade: Updating osd.5 2024-09-17T12:06:14.382 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:14 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:14.382 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:14 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-17T12:06:14.382 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:14 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:06:14.382 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:14 smithi165 ceph-mon[49373]: Deploying daemon osd.5 on smithi165 2024-09-17T12:06:14.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 225571 9.46 MB/sec execute 495 sec latency 80.848 ms 2024-09-17T12:06:15.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 225894 9.47 MB/sec execute 496 sec latency 116.504 ms 2024-09-17T12:06:16.091 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:16 smithi045 ceph-mon[63880]: pgmap v214: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 3.5 MiB/s wr, 405 op/s; 56 KiB/s, 1 objects/s recovering 2024-09-17T12:06:16.364 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:16 smithi165 ceph-mon[49373]: pgmap v214: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 3.5 MiB/s wr, 405 op/s; 56 KiB/s, 1 objects/s recovering 2024-09-17T12:06:16.365 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:16 smithi165 systemd[1]: Stopping Ceph osd.5 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:06:16.765 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:16 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[32880]: 2024-09-17T12:06:16.397+0000 7f6f1d834700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T12:06:16.765 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:16 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[32880]: 2024-09-17T12:06:16.397+0000 7f6f1d834700 -1 osd.5 121 *** Got signal Terminated *** 2024-09-17T12:06:16.765 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:16 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[32880]: 2024-09-17T12:06:16.397+0000 7f6f1d834700 -1 osd.5 121 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T12:06:16.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 226187 9.47 MB/sec execute 497 sec latency 43.552 ms 2024-09-17T12:06:17.122 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:17 smithi045 ceph-mon[63880]: osd.5 marked itself down and dead 2024-09-17T12:06:17.123 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:17 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:17.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:17 smithi165 ceph-mon[49373]: osd.5 marked itself down and dead 2024-09-17T12:06:17.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:17 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:17.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 226397 9.45 MB/sec execute 498 sec latency 671.336 ms 2024-09-17T12:06:17.989 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:17 smithi165 podman[54785]: 2024-09-17 12:06:17.677951407 +0000 UTC m=+1.409510248 container died 5d900eb9165bd9a81f7bd7b515676520fea52abb242e8d0a187d6045b683f8c1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, distribution-scope=public, io.openshift.tags=base centos centos-stream, version=8, io.buildah.version=1.19.8, name=centos-stream, release=754, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8) 2024-09-17T12:06:18.160 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:18 smithi045 ceph-mon[63880]: pgmap v215: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.1 MiB/s wr, 364 op/s; 51 KiB/s, 1 objects/s recovering 2024-09-17T12:06:18.160 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:18 smithi045 ceph-mon[63880]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T12:06:18.160 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:18 smithi045 ceph-mon[63880]: osdmap e122: 8 total, 7 up, 8 in 2024-09-17T12:06:18.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:18 smithi165 ceph-mon[49373]: pgmap v215: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.1 MiB/s wr, 364 op/s; 51 KiB/s, 1 objects/s recovering 2024-09-17T12:06:18.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:18 smithi165 ceph-mon[49373]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T12:06:18.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:18 smithi165 ceph-mon[49373]: osdmap e122: 8 total, 7 up, 8 in 2024-09-17T12:06:18.266 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:18 smithi165 podman[54785]: 2024-09-17 12:06:18.187474068 +0000 UTC m=+1.919032895 container cleanup 5d900eb9165bd9a81f7bd7b515676520fea52abb242e8d0a187d6045b683f8c1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5, release=754, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, RELEASE=HEAD, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, version=8, vcs-type=git, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, ceph=True) 2024-09-17T12:06:18.266 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:18 smithi165 bash[54785]: ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5 2024-09-17T12:06:18.765 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:18 smithi165 podman[54801]: 2024-09-17 12:06:18.447555715 +0000 UTC m=+0.765754366 container remove 5d900eb9165bd9a81f7bd7b515676520fea52abb242e8d0a187d6045b683f8c1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.tags=base centos centos-stream, release=754, ceph=True) 2024-09-17T12:06:18.772 INFO:teuthology.orchestra.run.smithi045.stdout: 5 226678 9.44 MB/sec execute 499 sec latency 981.630 ms 2024-09-17T12:06:19.137 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:19 smithi165 ceph-mon[49373]: osdmap e123: 8 total, 7 up, 8 in 2024-09-17T12:06:19.138 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:18 smithi165 podman[54887]: 2024-09-17 12:06:18.762446852 +0000 UTC m=+0.018465492 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:06:19.138 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:18 smithi165 podman[54887]: 2024-09-17 12:06:18.881927159 +0000 UTC m=+0.137945801 container create 67c952f32ece415359731577f7c5035a6a7dcf51e33a51da28ef1e5b5ccbf29a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-deactivate, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=main-7498493, org.label-schema.vendor=CentOS) 2024-09-17T12:06:19.169 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:19 smithi045 ceph-mon[63880]: osdmap e123: 8 total, 7 up, 8 in 2024-09-17T12:06:19.486 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:19 smithi165 podman[54887]: 2024-09-17 12:06:19.268661394 +0000 UTC m=+0.524680031 container init 67c952f32ece415359731577f7c5035a6a7dcf51e33a51da28ef1e5b5ccbf29a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=main-7498493, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T12:06:19.486 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:19 smithi165 podman[54887]: 2024-09-17 12:06:19.272694075 +0000 UTC m=+0.528712707 container start 67c952f32ece415359731577f7c5035a6a7dcf51e33a51da28ef1e5b5ccbf29a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, RELEASE=main-7498493, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True) 2024-09-17T12:06:19.486 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:19 smithi165 podman[54887]: 2024-09-17 12:06:19.360159483 +0000 UTC m=+0.616178111 container attach 67c952f32ece415359731577f7c5035a6a7dcf51e33a51da28ef1e5b5ccbf29a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=main-7498493, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-17T12:06:19.486 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:19 smithi165 podman[54887]: 2024-09-17 12:06:19.485233141 +0000 UTC m=+0.741251771 container died 67c952f32ece415359731577f7c5035a6a7dcf51e33a51da28ef1e5b5ccbf29a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-deactivate, RELEASE=main-7498493, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-17T12:06:19.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 226912 9.43 MB/sec execute 500 sec latency 71.750 ms 2024-09-17T12:06:20.149 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:20 smithi165 ceph-mon[49373]: pgmap v218: 97 pgs: 1 active+undersized, 4 peering, 7 stale+active+clean, 7 active+undersized+degraded, 78 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 5.3 MiB/s wr, 463 op/s; 72/1680 objects degraded (4.286%) 2024-09-17T12:06:20.149 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:20 smithi165 ceph-mon[49373]: Health check failed: Degraded data redundancy: 72/1680 objects degraded (4.286%), 7 pgs degraded (PG_DEGRADED) 2024-09-17T12:06:20.208 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:20 smithi045 ceph-mon[63880]: pgmap v218: 97 pgs: 1 active+undersized, 4 peering, 7 stale+active+clean, 7 active+undersized+degraded, 78 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 5.3 MiB/s wr, 463 op/s; 72/1680 objects degraded (4.286%) 2024-09-17T12:06:20.208 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:20 smithi045 ceph-mon[63880]: Health check failed: Degraded data redundancy: 72/1680 objects degraded (4.286%), 7 pgs degraded (PG_DEGRADED) 2024-09-17T12:06:20.399 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:20 smithi165 podman[54910]: 2024-09-17 12:06:20.147960245 +0000 UTC m=+0.656387762 container remove 67c952f32ece415359731577f7c5035a6a7dcf51e33a51da28ef1e5b5ccbf29a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, RELEASE=main-7498493, org.label-schema.license=GPLv2) 2024-09-17T12:06:20.399 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:20 smithi165 systemd[1]: ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.5.service: Deactivated successfully. 2024-09-17T12:06:20.399 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:20 smithi165 systemd[1]: Stopped Ceph osd.5 for a12537de-74ea-11ef-bceb-c7b262605968. 2024-09-17T12:06:20.399 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:20 smithi165 systemd[1]: ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.5.service: Consumed 4min 19.328s CPU time. 2024-09-17T12:06:20.399 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:20 smithi165 systemd[1]: Starting Ceph osd.5 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:06:20.650 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:20 smithi165 podman[55010]: 2024-09-17 12:06:20.523404793 +0000 UTC m=+0.016918765 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:06:20.650 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:20 smithi165 podman[55010]: 2024-09-17 12:06:20.649408721 +0000 UTC m=+0.142922676 container create 831addc646dee46f242c8a4b3f929007dfacfe2929c02f925102a7a8e05344d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493) 2024-09-17T12:06:20.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 227200 9.43 MB/sec execute 501 sec latency 124.212 ms 2024-09-17T12:06:21.265 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:20 smithi165 podman[55010]: 2024-09-17 12:06:20.952539844 +0000 UTC m=+0.446053809 container init 831addc646dee46f242c8a4b3f929007dfacfe2929c02f925102a7a8e05344d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=main-7498493, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T12:06:21.265 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:20 smithi165 podman[55010]: 2024-09-17 12:06:20.958089709 +0000 UTC m=+0.451603667 container start 831addc646dee46f242c8a4b3f929007dfacfe2929c02f925102a7a8e05344d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=main-7498493, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-17T12:06:21.265 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:21 smithi165 podman[55010]: 2024-09-17 12:06:21.043962791 +0000 UTC m=+0.537476745 container attach 831addc646dee46f242c8a4b3f929007dfacfe2929c02f925102a7a8e05344d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-activate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-17T12:06:21.266 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:21 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-activate[55024]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:06:21.266 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:21 smithi165 bash[55010]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:06:21.266 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:21 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-activate[55024]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:06:21.266 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:21 smithi165 bash[55010]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:06:21.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 227532 9.43 MB/sec execute 502 sec latency 80.749 ms 2024-09-17T12:06:22.269 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:22 smithi045 ceph-mon[63880]: pgmap v219: 97 pgs: 3 active+undersized, 4 peering, 6 stale+active+clean, 9 active+undersized+degraded, 75 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.6 MiB/s wr, 446 op/s; 78/1680 objects degraded (4.643%) 2024-09-17T12:06:22.269 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:22 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:22.269 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:22 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:06:22.499 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:22 smithi165 ceph-mon[49373]: pgmap v219: 97 pgs: 3 active+undersized, 4 peering, 6 stale+active+clean, 9 active+undersized+degraded, 75 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.6 MiB/s wr, 446 op/s; 78/1680 objects degraded (4.643%) 2024-09-17T12:06:22.499 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:22 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:22.499 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:22 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:06:22.500 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:22 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-activate[55024]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T12:06:22.500 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:22 smithi165 bash[55010]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T12:06:22.500 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:22 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-activate[55024]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T12:06:22.500 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:22 smithi165 bash[55010]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T12:06:22.500 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:22 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-activate[55024]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T12:06:22.500 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:22 smithi165 bash[55010]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T12:06:22.766 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:22 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-activate[55024]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T12:06:22.766 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:22 smithi165 bash[55010]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T12:06:22.766 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:22 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-activate[55024]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-17T12:06:22.766 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:22 smithi165 bash[55010]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-17T12:06:22.766 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:22 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-activate[55024]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T12:06:22.766 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:22 smithi165 bash[55010]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T12:06:22.766 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:22 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-activate[55024]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-17T12:06:22.766 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:22 smithi165 bash[55010]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-17T12:06:22.766 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:22 smithi165 podman[55010]: 2024-09-17 12:06:22.531184089 +0000 UTC m=+2.024698047 container died 831addc646dee46f242c8a4b3f929007dfacfe2929c02f925102a7a8e05344d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-17T12:06:22.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 227948 9.43 MB/sec execute 503 sec latency 44.618 ms 2024-09-17T12:06:23.401 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:23 smithi165 podman[55162]: 2024-09-17 12:06:23.148929982 +0000 UTC m=+0.611140999 container remove 831addc646dee46f242c8a4b3f929007dfacfe2929c02f925102a7a8e05344d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-activate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=main-7498493, CEPH_POINT_RELEASE=) 2024-09-17T12:06:23.401 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:23 smithi165 podman[55236]: 2024-09-17 12:06:23.262583094 +0000 UTC m=+0.017202562 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:06:23.401 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:23 smithi165 podman[55236]: 2024-09-17 12:06:23.399848628 +0000 UTC m=+0.154468068 container create b9ade3d65856493d1283258db5ad7a33d7bc28668473f732d75595469a5cecba (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5, org.label-schema.vendor=CentOS, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T12:06:23.662 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:23 smithi165 podman[55236]: 2024-09-17 12:06:23.63592512 +0000 UTC m=+0.390544580 container init b9ade3d65856493d1283258db5ad7a33d7bc28668473f732d75595469a5cecba (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=main-7498493, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-17T12:06:23.662 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:23 smithi165 podman[55236]: 2024-09-17 12:06:23.639443729 +0000 UTC m=+0.394063171 container start b9ade3d65856493d1283258db5ad7a33d7bc28668473f732d75595469a5cecba (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True) 2024-09-17T12:06:23.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 228232 9.42 MB/sec execute 504 sec latency 51.792 ms 2024-09-17T12:06:24.015 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:23 smithi165 ceph-osd[55255]: -- 172.21.15.165:0/2012142409 <== mon.1 v2:172.21.15.165:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ad0f30e1e0 con 0x55ad0f29a000 2024-09-17T12:06:24.015 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:23 smithi165 bash[55236]: b9ade3d65856493d1283258db5ad7a33d7bc28668473f732d75595469a5cecba 2024-09-17T12:06:24.016 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:23 smithi165 systemd[1]: Started Ceph osd.5 for a12537de-74ea-11ef-bceb-c7b262605968. 2024-09-17T12:06:24.055 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:24 smithi045 ceph-mon[63880]: pgmap v220: 97 pgs: 8 active+undersized, 4 peering, 20 active+undersized+degraded, 65 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.0 MiB/s wr, 403 op/s; 143/1590 objects degraded (8.994%) 2024-09-17T12:06:24.055 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:24 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:24.055 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:24 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:24.055 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:24 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:06:24.427 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:24 smithi165 ceph-mon[49373]: pgmap v220: 97 pgs: 8 active+undersized, 4 peering, 20 active+undersized+degraded, 65 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.0 MiB/s wr, 403 op/s; 143/1590 objects degraded (8.994%) 2024-09-17T12:06:24.427 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:24 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:24.428 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:24 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:24.428 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:24 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:06:24.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 228483 9.42 MB/sec execute 505 sec latency 45.424 ms 2024-09-17T12:06:25.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:25 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 170/1569 objects degraded (10.835%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T12:06:25.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:25 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 170/1569 objects degraded (10.835%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T12:06:25.765 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:25 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:06:25.523+0000 7f1bcade8640 -1 Falling back to public interface 2024-09-17T12:06:25.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 228739 9.42 MB/sec execute 506 sec latency 72.152 ms 2024-09-17T12:06:26.109 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:26 smithi045 ceph-mon[63880]: pgmap v221: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.1 MiB/s wr, 477 op/s; 170/1569 objects degraded (10.835%) 2024-09-17T12:06:26.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:26 smithi165 ceph-mon[49373]: pgmap v221: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.1 MiB/s wr, 477 op/s; 170/1569 objects degraded (10.835%) 2024-09-17T12:06:26.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 229005 9.42 MB/sec execute 507 sec latency 61.269 ms 2024-09-17T12:06:27.003 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T12:06:27.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 229350 9.42 MB/sec execute 508 sec latency 66.233 ms 2024-09-17T12:06:28.032 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T12:06:28.032 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (7m) 76s ago 17m 20.4M - 0.27.0 11f11916f8cd 2a2a54402d2a 2024-09-17T12:06:28.032 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (4m) 76s ago 16m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 e545cbf7736d 2024-09-17T12:06:28.033 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (4m) 0s ago 16m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 c1e89e114e4d 2024-09-17T12:06:28.033 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (7m) 76s ago 16m 57.0M - dad864ee21e9 371aaf55379f 2024-09-17T12:06:28.033 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (11m) 76s ago 11m 37.9M - 17.2.0 e1d6a67b021e 6a2a4a998dd1 2024-09-17T12:06:28.033 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (10m) 0s ago 10m 16.9M - 17.2.0 e1d6a67b021e d4c2987b9bac 2024-09-17T12:06:28.033 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:8443,9283,8765 running (6m) 76s ago 18m 520M - 19.3.0-4937-g74984932 6c1fcab85602 ae2f507bd67d 2024-09-17T12:06:28.033 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283,8765 running (6m) 0s ago 16m 457M - 19.3.0-4937-g74984932 6c1fcab85602 53403866652e 2024-09-17T12:06:28.033 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (5m) 76s ago 18m 52.8M 2048M 19.3.0-4937-g74984932 6c1fcab85602 737eec78e0f0 2024-09-17T12:06:28.033 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (5m) 0s ago 16m 44.1M 2048M 19.3.0-4937-g74984932 6c1fcab85602 249eedb4123e 2024-09-17T12:06:28.033 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (10m) 76s ago 10m 279M - 4.0 e1d6a67b021e b8fc749ebb71 2024-09-17T12:06:28.033 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (10m) 0s ago 10m 86.5M - 4.0 e1d6a67b021e 771b27764432 2024-09-17T12:06:28.033 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (7m) 76s ago 16m 11.1M - 1.7.0 72c9c2088986 989b5738cb69 2024-09-17T12:06:28.033 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (7m) 0s ago 16m 12.1M - 1.7.0 72c9c2088986 e1e8ffab3f21 2024-09-17T12:06:28.033 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (3m) 76s ago 15m 338M 1937M 19.3.0-4937-g74984932 6c1fcab85602 20dafd01bd12 2024-09-17T12:06:28.033 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (3m) 76s ago 15m 264M 1937M 19.3.0-4937-g74984932 6c1fcab85602 d39be5014418 2024-09-17T12:06:28.034 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (2m) 76s ago 14m 187M 1937M 19.3.0-4937-g74984932 6c1fcab85602 e13c2ecd369a 2024-09-17T12:06:28.034 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (85s) 76s ago 14m 12.1M 1937M 19.3.0-4937-g74984932 6c1fcab85602 ffd68312d242 2024-09-17T12:06:28.034 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (41s) 0s ago 13m 256M 2704M 19.3.0-4937-g74984932 6c1fcab85602 589de85559fd 2024-09-17T12:06:28.035 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (4s) 0s ago 13m 13.3M 2704M 19.3.0-4937-g74984932 6c1fcab85602 b9ade3d65856 2024-09-17T12:06:28.035 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (12m) 0s ago 12m 465M 2704M 17.2.0 e1d6a67b021e fae3e1bae466 2024-09-17T12:06:28.035 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (12m) 0s ago 12m 744M 2704M 17.2.0 e1d6a67b021e 71b9f2cdb810 2024-09-17T12:06:28.035 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (6m) 76s ago 15m 56.0M - 2.51.0 1d3b7f56885b 61b064ad6609 2024-09-17T12:06:28.188 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:28 smithi045 ceph-mon[63880]: pgmap v222: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.2 MiB/s wr, 321 op/s; 170/1569 objects degraded (10.835%) 2024-09-17T12:06:28.188 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:28 smithi045 ceph-mon[63880]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:06:28.188 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:28 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:28.188 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:28 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:28.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:28 smithi165 ceph-mon[49373]: pgmap v222: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 3.2 MiB/s wr, 321 op/s; 170/1569 objects degraded (10.835%) 2024-09-17T12:06:28.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:28 smithi165 ceph-mon[49373]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:06:28.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:28 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:28.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:28 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:28.622 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:06:28.622 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T12:06:28.622 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:06:28.622 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:06:28.622 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T12:06:28.622 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:06:28.622 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:06:28.622 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T12:06:28.623 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-17T12:06:28.623 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 5 2024-09-17T12:06:28.623 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:06:28.623 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T12:06:28.623 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T12:06:28.623 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:06:28.623 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T12:06:28.623 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-17T12:06:28.623 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 9 2024-09-17T12:06:28.623 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T12:06:28.623 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:06:28.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 229604 9.43 MB/sec execute 509 sec latency 61.942 ms 2024-09-17T12:06:29.193 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:06:29.193 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T12:06:29.193 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T12:06:29.193 INFO:teuthology.orchestra.run.smithi045.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T12:06:29.193 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [ 2024-09-17T12:06:29.193 INFO:teuthology.orchestra.run.smithi045.stdout: "crash", 2024-09-17T12:06:29.193 INFO:teuthology.orchestra.run.smithi045.stdout: "mon", 2024-09-17T12:06:29.194 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr" 2024-09-17T12:06:29.194 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-17T12:06:29.194 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "12/23 daemons upgraded", 2024-09-17T12:06:29.194 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T12:06:29.194 INFO:teuthology.orchestra.run.smithi045.stdout: "is_paused": false 2024-09-17T12:06:29.194 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:06:29.218 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:29 smithi045 ceph-mon[63880]: from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:06:29.218 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:29 smithi045 ceph-mon[63880]: from='client.44299 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:06:29.218 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:29 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:29.218 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:29 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:29.218 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:29 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/4130798265' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:06:29.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:29 smithi165 ceph-mon[49373]: from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:06:29.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:29 smithi165 ceph-mon[49373]: from='client.44299 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:06:29.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:29 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:29.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:29 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:29.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:29 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/4130798265' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:06:29.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 229851 9.42 MB/sec execute 510 sec latency 50.485 ms 2024-09-17T12:06:29.832 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 167/1632 objects degraded (10.233%), 23 pgs degraded 2024-09-17T12:06:29.833 INFO:teuthology.orchestra.run.smithi045.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T12:06:29.833 INFO:teuthology.orchestra.run.smithi045.stdout: osd.5 (root=default,host=smithi165) is down 2024-09-17T12:06:29.833 INFO:teuthology.orchestra.run.smithi045.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 167/1632 objects degraded (10.233%), 23 pgs degraded 2024-09-17T12:06:29.833 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-09-17T12:06:29.833 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.3 is active+undersized+degraded, acting [2,7] 2024-09-17T12:06:29.833 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.b is active+undersized+degraded, acting [7,4] 2024-09-17T12:06:29.833 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.c is active+undersized+degraded, acting [2,0] 2024-09-17T12:06:29.833 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.12 is active+undersized+degraded, acting [3,7] 2024-09-17T12:06:29.833 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.14 is active+undersized+degraded, acting [6,3] 2024-09-17T12:06:29.833 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.15 is active+undersized+degraded, acting [1,0] 2024-09-17T12:06:29.833 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-09-17T12:06:29.833 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.17 is active+undersized+degraded, acting [6,2] 2024-09-17T12:06:29.834 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.18 is active+undersized+degraded, acting [4,7] 2024-09-17T12:06:29.834 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.1c is active+undersized+degraded, acting [7,2] 2024-09-17T12:06:29.834 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.1e is active+undersized+degraded, acting [3,0] 2024-09-17T12:06:29.834 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.2 is active+undersized+degraded, acting [3,6] 2024-09-17T12:06:29.834 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.4 is active+undersized+degraded, acting [1,2] 2024-09-17T12:06:29.834 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-09-17T12:06:29.834 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.c is active+undersized+degraded, acting [3,6] 2024-09-17T12:06:29.834 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.d is active+undersized+degraded, acting [7,6] 2024-09-17T12:06:29.834 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-09-17T12:06:29.834 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.16 is active+undersized+degraded, acting [7,1] 2024-09-17T12:06:29.834 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.17 is active+undersized+degraded, acting [0,3] 2024-09-17T12:06:29.834 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.1c is active+undersized+degraded, acting [4,1] 2024-09-17T12:06:29.834 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-09-17T12:06:29.834 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-09-17T12:06:30.243 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:30 smithi045 ceph-mon[63880]: pgmap v223: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.0 MiB/s wr, 466 op/s; 167/1632 objects degraded (10.233%) 2024-09-17T12:06:30.243 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:30 smithi045 ceph-mon[63880]: from='client.34402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:06:30.243 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:30 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/4287201632' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:06:30.376 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:30 smithi165 ceph-mon[49373]: pgmap v223: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.0 MiB/s wr, 466 op/s; 167/1632 objects degraded (10.233%) 2024-09-17T12:06:30.376 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:30 smithi165 ceph-mon[49373]: from='client.34402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:06:30.376 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:30 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/4287201632' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:06:30.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 230191 9.40 MB/sec execute 511 sec latency 54.087 ms 2024-09-17T12:06:31.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 230562 9.40 MB/sec execute 512 sec latency 30.768 ms 2024-09-17T12:06:32.264 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:32 smithi045 ceph-mon[63880]: pgmap v224: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 3.9 MiB/s wr, 403 op/s; 167/1632 objects degraded (10.233%) 2024-09-17T12:06:32.432 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:32 smithi165 ceph-mon[49373]: pgmap v224: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 3.9 MiB/s wr, 403 op/s; 167/1632 objects degraded (10.233%) 2024-09-17T12:06:32.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 231088 9.41 MB/sec execute 513 sec latency 43.347 ms 2024-09-17T12:06:33.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 231603 9.42 MB/sec execute 514 sec latency 35.893 ms 2024-09-17T12:06:33.779 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:33 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:33.779 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:33 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:33.779 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:33 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:06:33.779 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:33 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:06:33.779 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:33 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:33.779 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:33 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:33.779 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:33 smithi045 ceph-mon[63880]: pgmap v225: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.8 MiB/s wr, 526 op/s; 175/1680 objects degraded (10.417%) 2024-09-17T12:06:33.779 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:33 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 167/1632 objects degraded (10.233%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T12:06:33.780 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:33 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:06:33.780 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:33 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:06:33.780 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:33 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:06:33.780 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:33 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:06:33.780 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:33 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-17T12:06:34.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:33 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:34.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:33 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:34.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:33 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:06:34.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:33 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:06:34.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:33 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:34.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:33 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:34.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:33 smithi165 ceph-mon[49373]: pgmap v225: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 344 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.8 MiB/s wr, 526 op/s; 175/1680 objects degraded (10.417%) 2024-09-17T12:06:34.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:33 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 167/1632 objects degraded (10.233%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T12:06:34.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:33 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:06:34.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:33 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:06:34.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:33 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:06:34.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:33 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:06:34.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:33 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-17T12:06:34.765 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:34 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:06:34.426+0000 7f1bcade8640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-17T12:06:34.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 231969 9.42 MB/sec execute 515 sec latency 63.798 ms 2024-09-17T12:06:34.801 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:34 smithi045 ceph-mon[63880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T12:06:34.801 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:34 smithi045 ceph-mon[63880]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-17T12:06:35.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:34 smithi165 ceph-mon[49373]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T12:06:35.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:34 smithi165 ceph-mon[49373]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-17T12:06:35.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 232307 9.42 MB/sec execute 516 sec latency 69.711 ms 2024-09-17T12:06:35.829 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:35 smithi045 ceph-mon[63880]: pgmap v226: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 4.8 MiB/s wr, 455 op/s; 169/1644 objects degraded (10.280%) 2024-09-17T12:06:35.829 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:35 smithi045 ceph-mon[63880]: from='osd.5 [v2:172.21.15.165:6808/2025241981,v1:172.21.15.165:6809/2025241981]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-17T12:06:35.829 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:35 smithi045 ceph-mon[63880]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-17T12:06:36.015 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:35 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:06:35.589+0000 7f1bcade8640 -1 osd.5 121 log_to_monitors true 2024-09-17T12:06:36.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:35 smithi165 ceph-mon[49373]: pgmap v226: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 4.8 MiB/s wr, 455 op/s; 169/1644 objects degraded (10.280%) 2024-09-17T12:06:36.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:35 smithi165 ceph-mon[49373]: from='osd.5 [v2:172.21.15.165:6808/2025241981,v1:172.21.15.165:6809/2025241981]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-17T12:06:36.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:35 smithi165 ceph-mon[49373]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-17T12:06:36.765 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:06:36 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:06:36.277+0000 7f1bbeb8a640 -1 osd.5 121 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T12:06:36.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 232500 9.41 MB/sec execute 517 sec latency 114.938 ms 2024-09-17T12:06:36.902 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:36 smithi045 ceph-mon[63880]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T12:06:36.902 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:36 smithi045 ceph-mon[63880]: from='osd.5 [v2:172.21.15.165:6808/2025241981,v1:172.21.15.165:6809/2025241981]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi165", "root=default"]} : dispatch 2024-09-17T12:06:36.902 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:36 smithi045 ceph-mon[63880]: osdmap e124: 8 total, 7 up, 8 in 2024-09-17T12:06:36.902 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:36 smithi045 ceph-mon[63880]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi165", "root=default"]} : dispatch 2024-09-17T12:06:36.902 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:36 smithi045 ceph-mon[63880]: from='osd.5 ' entity='osd.5' 2024-09-17T12:06:36.902 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:06:37.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:36 smithi165 ceph-mon[49373]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T12:06:37.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:36 smithi165 ceph-mon[49373]: from='osd.5 [v2:172.21.15.165:6808/2025241981,v1:172.21.15.165:6809/2025241981]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi165", "root=default"]} : dispatch 2024-09-17T12:06:37.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:36 smithi165 ceph-mon[49373]: osdmap e124: 8 total, 7 up, 8 in 2024-09-17T12:06:37.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:36 smithi165 ceph-mon[49373]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi165", "root=default"]} : dispatch 2024-09-17T12:06:37.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:36 smithi165 ceph-mon[49373]: from='osd.5 ' entity='osd.5' 2024-09-17T12:06:37.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:06:37.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 232706 9.41 MB/sec execute 518 sec latency 502.051 ms 2024-09-17T12:06:38.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:37 smithi045 ceph-mon[63880]: pgmap v228: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 341 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.9 MiB/s wr, 487 op/s; 169/1644 objects degraded (10.280%) 2024-09-17T12:06:38.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:37 smithi045 ceph-mon[63880]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T12:06:38.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:37 smithi045 ceph-mon[63880]: osd.5 [v2:172.21.15.165:6808/2025241981,v1:172.21.15.165:6809/2025241981] boot 2024-09-17T12:06:38.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:37 smithi045 ceph-mon[63880]: osdmap e125: 8 total, 8 up, 8 in 2024-09-17T12:06:38.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:37 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-17T12:06:38.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:37 smithi165 ceph-mon[49373]: pgmap v228: 97 pgs: 9 active+undersized, 23 active+undersized+degraded, 65 active+clean; 341 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.9 MiB/s wr, 487 op/s; 169/1644 objects degraded (10.280%) 2024-09-17T12:06:38.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:37 smithi165 ceph-mon[49373]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T12:06:38.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:37 smithi165 ceph-mon[49373]: osd.5 [v2:172.21.15.165:6808/2025241981,v1:172.21.15.165:6809/2025241981] boot 2024-09-17T12:06:38.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:37 smithi165 ceph-mon[49373]: osdmap e125: 8 total, 8 up, 8 in 2024-09-17T12:06:38.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:37 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-17T12:06:38.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 232773 9.40 MB/sec execute 519 sec latency 1075.105 ms 2024-09-17T12:06:38.874 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:38 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 169/1644 objects degraded (10.280%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T12:06:38.874 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:38 smithi045 ceph-mon[63880]: osdmap e126: 8 total, 8 up, 8 in 2024-09-17T12:06:39.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:38 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 169/1644 objects degraded (10.280%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T12:06:39.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:38 smithi165 ceph-mon[49373]: osdmap e126: 8 total, 8 up, 8 in 2024-09-17T12:06:39.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 232917 9.39 MB/sec execute 520 sec latency 1087.610 ms 2024-09-17T12:06:39.862 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:39 smithi045 ceph-mon[63880]: pgmap v231: 97 pgs: 1 remapped+peering, 4 peering, 7 active+undersized, 19 active+undersized+degraded, 66 active+clean; 343 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.0 MiB/s wr, 367 op/s; 123/1590 objects degraded (7.736%) 2024-09-17T12:06:39.863 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:39 smithi045 ceph-mon[63880]: osdmap e127: 8 total, 8 up, 8 in 2024-09-17T12:06:40.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:39 smithi165 ceph-mon[49373]: pgmap v231: 97 pgs: 1 remapped+peering, 4 peering, 7 active+undersized, 19 active+undersized+degraded, 66 active+clean; 343 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.0 MiB/s wr, 367 op/s; 123/1590 objects degraded (7.736%) 2024-09-17T12:06:40.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:39 smithi165 ceph-mon[49373]: osdmap e127: 8 total, 8 up, 8 in 2024-09-17T12:06:40.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 233148 9.38 MB/sec execute 521 sec latency 504.847 ms 2024-09-17T12:06:41.524 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:41 smithi045 ceph-mon[63880]: osdmap e128: 8 total, 8 up, 8 in 2024-09-17T12:06:41.524 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:41 smithi045 ceph-mon[63880]: pgmap v234: 97 pgs: 1 remapped+peering, 7 peering, 2 active+undersized, 10 active+undersized+degraded, 77 active+clean; 344 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.6 MiB/s wr, 579 op/s; 69/1602 objects degraded (4.307%); 258 KiB/s, 11 objects/s recovering 2024-09-17T12:06:41.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:41 smithi165 ceph-mon[49373]: osdmap e128: 8 total, 8 up, 8 in 2024-09-17T12:06:41.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:41 smithi165 ceph-mon[49373]: pgmap v234: 97 pgs: 1 remapped+peering, 7 peering, 2 active+undersized, 10 active+undersized+degraded, 77 active+clean; 344 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.6 MiB/s wr, 579 op/s; 69/1602 objects degraded (4.307%); 258 KiB/s, 11 objects/s recovering 2024-09-17T12:06:41.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 233270 9.38 MB/sec execute 522 sec latency 1028.206 ms 2024-09-17T12:06:42.300 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:42 smithi045 ceph-mon[63880]: osdmap e129: 8 total, 8 up, 8 in 2024-09-17T12:06:42.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:42 smithi165 ceph-mon[49373]: osdmap e129: 8 total, 8 up, 8 in 2024-09-17T12:06:42.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 233503 9.37 MB/sec execute 523 sec latency 850.306 ms 2024-09-17T12:06:43.312 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:43 smithi045 ceph-mon[63880]: osdmap e130: 8 total, 8 up, 8 in 2024-09-17T12:06:43.312 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:43 smithi045 ceph-mon[63880]: pgmap v237: 97 pgs: 1 remapped+peering, 8 peering, 2 active+undersized+degraded, 86 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.2 MiB/s wr, 434 op/s; 21/1620 objects degraded (1.296%); 1.3 MiB/s, 30 objects/s recovering 2024-09-17T12:06:43.312 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:43 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 69/1602 objects degraded (4.307%), 10 pgs degraded (PG_DEGRADED) 2024-09-17T12:06:43.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:43 smithi165 ceph-mon[49373]: osdmap e130: 8 total, 8 up, 8 in 2024-09-17T12:06:43.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:43 smithi165 ceph-mon[49373]: pgmap v237: 97 pgs: 1 remapped+peering, 8 peering, 2 active+undersized+degraded, 86 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.2 MiB/s wr, 434 op/s; 21/1620 objects degraded (1.296%); 1.3 MiB/s, 30 objects/s recovering 2024-09-17T12:06:43.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:43 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 69/1602 objects degraded (4.307%), 10 pgs degraded (PG_DEGRADED) 2024-09-17T12:06:43.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 233836 9.36 MB/sec execute 524 sec latency 37.842 ms 2024-09-17T12:06:44.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 234047 9.35 MB/sec execute 525 sec latency 71.196 ms 2024-09-17T12:06:45.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 234329 9.34 MB/sec execute 526 sec latency 58.853 ms 2024-09-17T12:06:46.193 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:46 smithi045 ceph-mon[63880]: pgmap v238: 97 pgs: 4 peering, 2 active+undersized+degraded, 91 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 2.5 MiB/s wr, 225 op/s; 21/1623 objects degraded (1.294%); 1.2 MiB/s, 34 objects/s recovering 2024-09-17T12:06:46.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:46 smithi165 ceph-mon[49373]: pgmap v238: 97 pgs: 4 peering, 2 active+undersized+degraded, 91 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 2.5 MiB/s wr, 225 op/s; 21/1623 objects degraded (1.294%); 1.2 MiB/s, 34 objects/s recovering 2024-09-17T12:06:46.776 INFO:teuthology.orchestra.run.smithi045.stdout: 5 234639 9.34 MB/sec execute 527 sec latency 63.248 ms 2024-09-17T12:06:47.215 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:47 smithi045 ceph-mon[63880]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/1623 objects degraded (1.294%), 2 pgs degraded) 2024-09-17T12:06:47.215 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:47 smithi045 ceph-mon[63880]: Cluster is now healthy 2024-09-17T12:06:47.432 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:47 smithi165 ceph-mon[49373]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/1623 objects degraded (1.294%), 2 pgs degraded) 2024-09-17T12:06:47.432 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:47 smithi165 ceph-mon[49373]: Cluster is now healthy 2024-09-17T12:06:47.773 INFO:teuthology.orchestra.run.smithi045.stdout: 5 235045 9.34 MB/sec execute 528 sec latency 42.217 ms 2024-09-17T12:06:48.272 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:48 smithi045 ceph-mon[63880]: pgmap v239: 97 pgs: 1 peering, 96 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.3 MiB/s wr, 210 op/s; 1.8 MiB/s, 40 objects/s recovering 2024-09-17T12:06:48.511 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:48 smithi165 ceph-mon[49373]: pgmap v239: 97 pgs: 1 peering, 96 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.3 MiB/s wr, 210 op/s; 1.8 MiB/s, 40 objects/s recovering 2024-09-17T12:06:48.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 235325 9.35 MB/sec execute 529 sec latency 59.190 ms 2024-09-17T12:06:49.175 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:49 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-17T12:06:49.304 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:49 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-17T12:06:49.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 235519 9.34 MB/sec execute 530 sec latency 43.805 ms 2024-09-17T12:06:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:50 smithi165 ceph-mon[49373]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T12:06:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:50 smithi165 ceph-mon[49373]: Upgrade: osd.6 is safe to restart 2024-09-17T12:06:50.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:50 smithi165 ceph-mon[49373]: pgmap v240: 97 pgs: 97 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 3.7 MiB/s wr, 399 op/s; 1.3 MiB/s, 28 objects/s recovering 2024-09-17T12:06:50.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:50 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:50.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:50 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-17T12:06:50.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:50 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:06:50.320 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:50 smithi045 ceph-mon[63880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T12:06:50.320 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:50 smithi045 ceph-mon[63880]: Upgrade: osd.6 is safe to restart 2024-09-17T12:06:50.320 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:50 smithi045 ceph-mon[63880]: pgmap v240: 97 pgs: 97 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 3.7 MiB/s wr, 399 op/s; 1.3 MiB/s, 28 objects/s recovering 2024-09-17T12:06:50.320 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:50.320 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-17T12:06:50.320 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:06:50.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 235892 9.34 MB/sec execute 531 sec latency 51.343 ms 2024-09-17T12:06:51.081 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:51 smithi045 ceph-mon[63880]: Upgrade: Updating osd.6 2024-09-17T12:06:51.081 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:51 smithi045 ceph-mon[63880]: Deploying daemon osd.6 on smithi165 2024-09-17T12:06:51.456 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:51 smithi165 ceph-mon[49373]: Upgrade: Updating osd.6 2024-09-17T12:06:51.456 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:51 smithi165 ceph-mon[49373]: Deploying daemon osd.6 on smithi165 2024-09-17T12:06:51.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 236189 9.34 MB/sec execute 532 sec latency 55.866 ms 2024-09-17T12:06:52.104 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:52 smithi045 ceph-mon[63880]: pgmap v241: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.3 MiB/s wr, 344 op/s; 1.1 MiB/s, 23 objects/s recovering 2024-09-17T12:06:52.104 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:52 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:52.104 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:52 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:06:52.169 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:52 smithi165 ceph-mon[49373]: pgmap v241: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.3 MiB/s wr, 344 op/s; 1.1 MiB/s, 23 objects/s recovering 2024-09-17T12:06:52.169 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:52 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:52.169 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:52 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:06:52.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 236454 9.34 MB/sec execute 533 sec latency 64.558 ms 2024-09-17T12:06:53.639 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:53 smithi165 systemd[1]: Stopping Ceph osd.6 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:06:53.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 236650 9.33 MB/sec execute 534 sec latency 58.645 ms 2024-09-17T12:06:54.015 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:53 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[35853]: 2024-09-17T12:06:53.637+0000 7f5d5889c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T12:06:54.015 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:53 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[35853]: 2024-09-17T12:06:53.637+0000 7f5d5889c700 -1 osd.6 130 *** Got signal Terminated *** 2024-09-17T12:06:54.015 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:53 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[35853]: 2024-09-17T12:06:53.637+0000 7f5d5889c700 -1 osd.6 130 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T12:06:54.162 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:54 smithi045 ceph-mon[63880]: pgmap v242: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.4 MiB/s wr, 364 op/s; 629 KiB/s, 13 objects/s recovering 2024-09-17T12:06:54.162 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:54 smithi045 ceph-mon[63880]: osd.6 marked itself down and dead 2024-09-17T12:06:54.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:54 smithi165 ceph-mon[49373]: pgmap v242: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.4 MiB/s wr, 364 op/s; 629 KiB/s, 13 objects/s recovering 2024-09-17T12:06:54.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:54 smithi165 ceph-mon[49373]: osd.6 marked itself down and dead 2024-09-17T12:06:54.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 236889 9.33 MB/sec execute 535 sec latency 479.660 ms 2024-09-17T12:06:55.208 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:55 smithi045 ceph-mon[63880]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T12:06:55.208 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:55 smithi045 ceph-mon[63880]: osdmap e131: 8 total, 7 up, 8 in 2024-09-17T12:06:55.221 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:54 smithi165 podman[56803]: 2024-09-17 12:06:54.916227577 +0000 UTC m=+1.405303964 container died fae3e1bae466e09c0b6332ef8fba0fe659a59e789de4b873a5d2e45bda0fb449 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, distribution-scope=public, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., vcs-type=git, version=8, RELEASE=HEAD, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, ceph=True, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-17T12:06:55.221 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:55 smithi165 ceph-mon[49373]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T12:06:55.221 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:55 smithi165 ceph-mon[49373]: osdmap e131: 8 total, 7 up, 8 in 2024-09-17T12:06:55.515 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:55 smithi165 podman[56803]: 2024-09-17 12:06:55.312104427 +0000 UTC m=+1.801180824 container cleanup fae3e1bae466e09c0b6332ef8fba0fe659a59e789de4b873a5d2e45bda0fb449 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., io.buildah.version=1.19.8, io.openshift.expose-services=, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, maintainer=Guillaume Abrioux , RELEASE=HEAD, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-17T12:06:55.515 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:55 smithi165 bash[56803]: ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6 2024-09-17T12:06:55.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 237101 9.31 MB/sec execute 536 sec latency 1031.416 ms 2024-09-17T12:06:55.968 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:55 smithi165 podman[56817]: 2024-09-17 12:06:55.596337673 +0000 UTC m=+0.678447190 container remove fae3e1bae466e09c0b6332ef8fba0fe659a59e789de4b873a5d2e45bda0fb449 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, version=8, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, name=centos-stream, distribution-scope=public, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True) 2024-09-17T12:06:56.221 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:56 smithi165 ceph-mon[49373]: pgmap v244: 97 pgs: 10 stale+active+clean, 87 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.0 MiB/s wr, 382 op/s; 484 KiB/s, 7 objects/s recovering 2024-09-17T12:06:56.222 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:56 smithi165 ceph-mon[49373]: osdmap e132: 8 total, 7 up, 8 in 2024-09-17T12:06:56.222 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:55 smithi165 podman[56907]: 2024-09-17 12:06:55.870495568 +0000 UTC m=+0.016717930 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:06:56.222 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:56 smithi165 podman[56907]: 2024-09-17 12:06:56.072457547 +0000 UTC m=+0.218679899 container create 1563c349080517a321734226882bcda053cfa797d0589271ebda1c63533df61b (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=main-7498493, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-17T12:06:56.238 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:56 smithi045 ceph-mon[63880]: pgmap v244: 97 pgs: 10 stale+active+clean, 87 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.0 MiB/s wr, 382 op/s; 484 KiB/s, 7 objects/s recovering 2024-09-17T12:06:56.238 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:56 smithi045 ceph-mon[63880]: osdmap e132: 8 total, 7 up, 8 in 2024-09-17T12:06:56.766 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:56 smithi165 podman[56907]: 2024-09-17 12:06:56.333600309 +0000 UTC m=+0.479822678 container init 1563c349080517a321734226882bcda053cfa797d0589271ebda1c63533df61b (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493) 2024-09-17T12:06:56.766 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:56 smithi165 podman[56907]: 2024-09-17 12:06:56.338552106 +0000 UTC m=+0.484774460 container start 1563c349080517a321734226882bcda053cfa797d0589271ebda1c63533df61b (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T12:06:56.766 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:56 smithi165 podman[56907]: 2024-09-17 12:06:56.433488718 +0000 UTC m=+0.579711076 container attach 1563c349080517a321734226882bcda053cfa797d0589271ebda1c63533df61b (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=main-7498493, org.label-schema.vendor=CentOS) 2024-09-17T12:06:56.766 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:56 smithi165 podman[56907]: 2024-09-17 12:06:56.567504567 +0000 UTC m=+0.713726925 container died 1563c349080517a321734226882bcda053cfa797d0589271ebda1c63533df61b (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T12:06:56.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 237405 9.31 MB/sec execute 537 sec latency 56.518 ms 2024-09-17T12:06:57.515 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:57 smithi165 podman[56930]: 2024-09-17 12:06:57.221633754 +0000 UTC m=+0.647915895 container remove 1563c349080517a321734226882bcda053cfa797d0589271ebda1c63533df61b (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, RELEASE=main-7498493, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-17T12:06:57.515 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:57 smithi165 systemd[1]: ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.6.service: Deactivated successfully. 2024-09-17T12:06:57.515 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:57 smithi165 systemd[1]: Stopped Ceph osd.6 for a12537de-74ea-11ef-bceb-c7b262605968. 2024-09-17T12:06:57.516 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:57 smithi165 systemd[1]: ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.6.service: Consumed 3min 55.001s CPU time. 2024-09-17T12:06:57.534 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:57 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:57.534 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:57 smithi045 ceph-mon[63880]: pgmap v246: 97 pgs: 3 active+undersized, 8 stale+active+clean, 5 active+undersized+degraded, 81 active+clean; 348 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.5 MiB/s wr, 356 op/s; 30/1656 objects degraded (1.812%) 2024-09-17T12:06:57.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 237647 9.30 MB/sec execute 538 sec latency 82.224 ms 2024-09-17T12:06:57.848 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:57 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:06:57.848 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:57 smithi165 ceph-mon[49373]: pgmap v246: 97 pgs: 3 active+undersized, 8 stale+active+clean, 5 active+undersized+degraded, 81 active+clean; 348 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 3.5 MiB/s wr, 356 op/s; 30/1656 objects degraded (1.812%) 2024-09-17T12:06:57.849 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:57 smithi165 systemd[1]: Starting Ceph osd.6 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:06:57.849 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:57 smithi165 podman[57032]: 2024-09-17 12:06:57.659038104 +0000 UTC m=+0.019711224 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:06:58.103 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:57 smithi165 podman[57032]: 2024-09-17 12:06:57.847618585 +0000 UTC m=+0.208291704 container create 4bcb8a5b797a1bf1dfe734cd067bda695ab734261d19a4ea8ebdd38297c5c4e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-7498493, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0) 2024-09-17T12:06:58.516 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:58 smithi165 podman[57032]: 2024-09-17 12:06:58.102798187 +0000 UTC m=+0.463471300 container init 4bcb8a5b797a1bf1dfe734cd067bda695ab734261d19a4ea8ebdd38297c5c4e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=main-7498493, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-17T12:06:58.516 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:58 smithi165 podman[57032]: 2024-09-17 12:06:58.106750749 +0000 UTC m=+0.467423861 container start 4bcb8a5b797a1bf1dfe734cd067bda695ab734261d19a4ea8ebdd38297c5c4e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=main-7498493, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T12:06:58.516 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:58 smithi165 podman[57032]: 2024-09-17 12:06:58.202669333 +0000 UTC m=+0.563342448 container attach 4bcb8a5b797a1bf1dfe734cd067bda695ab734261d19a4ea8ebdd38297c5c4e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=main-7498493, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T12:06:58.516 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:58 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-activate[57046]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:06:58.516 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:58 smithi165 bash[57032]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:06:58.516 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:58 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-activate[57046]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:06:58.516 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:58 smithi165 bash[57032]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:06:58.542 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:58 smithi045 ceph-mon[63880]: Health check failed: Degraded data redundancy: 30/1656 objects degraded (1.812%), 5 pgs degraded (PG_DEGRADED) 2024-09-17T12:06:58.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 237974 9.30 MB/sec execute 539 sec latency 38.128 ms 2024-09-17T12:06:59.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:58 smithi165 ceph-mon[49373]: Health check failed: Degraded data redundancy: 30/1656 objects degraded (1.812%), 5 pgs degraded (PG_DEGRADED) 2024-09-17T12:06:59.352 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:59 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-activate[57046]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T12:06:59.352 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:59 smithi165 bash[57032]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T12:06:59.352 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:59 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-activate[57046]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T12:06:59.352 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:59 smithi165 bash[57032]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T12:06:59.547 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:06:59 smithi045 ceph-mon[63880]: pgmap v247: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.8 MiB/s wr, 484 op/s; 172/1542 objects degraded (11.154%) 2024-09-17T12:06:59.699 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:06:59 smithi165 ceph-mon[49373]: pgmap v247: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.8 MiB/s wr, 484 op/s; 172/1542 objects degraded (11.154%) 2024-09-17T12:06:59.700 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:59 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-activate[57046]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T12:06:59.700 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:59 smithi165 bash[57032]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T12:06:59.700 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:59 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-activate[57046]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T12:06:59.700 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:59 smithi165 bash[57032]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T12:06:59.700 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:59 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-activate[57046]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-17T12:06:59.700 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:59 smithi165 bash[57032]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-17T12:06:59.700 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:59 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-activate[57046]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T12:06:59.701 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:59 smithi165 bash[57032]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T12:06:59.701 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:59 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-activate[57046]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-17T12:06:59.701 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:59 smithi165 bash[57032]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-17T12:06:59.701 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:06:59 smithi165 podman[57032]: 2024-09-17 12:06:59.386322021 +0000 UTC m=+1.746995137 container died 4bcb8a5b797a1bf1dfe734cd067bda695ab734261d19a4ea8ebdd38297c5c4e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-7498493, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-17T12:06:59.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 238636 9.31 MB/sec execute 540 sec latency 50.925 ms 2024-09-17T12:07:00.439 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T12:07:00.559 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:07:00 smithi165 podman[57177]: 2024-09-17 12:07:00.307857009 +0000 UTC m=+0.910709353 container remove 4bcb8a5b797a1bf1dfe734cd067bda695ab734261d19a4ea8ebdd38297c5c4e4 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-activate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=main-7498493, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T12:07:00.560 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:07:00 smithi165 podman[57232]: 2024-09-17 12:07:00.432188536 +0000 UTC m=+0.019876803 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:07:00.560 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:07:00 smithi165 podman[57232]: 2024-09-17 12:07:00.558830686 +0000 UTC m=+0.146518942 container create 12ba3cdf09dd682cc7b2eab26abc4f76166fd7e25861ff8de5f2ff84f4273438 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6, RELEASE=main-7498493, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-17T12:07:00.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 239151 9.33 MB/sec execute 541 sec latency 25.224 ms 2024-09-17T12:07:00.822 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:07:00 smithi165 podman[57232]: 2024-09-17 12:07:00.794609541 +0000 UTC m=+0.382297806 container init 12ba3cdf09dd682cc7b2eab26abc4f76166fd7e25861ff8de5f2ff84f4273438 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=main-7498493, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-17T12:07:00.822 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:07:00 smithi165 podman[57232]: 2024-09-17 12:07:00.797945126 +0000 UTC m=+0.385633384 container start 12ba3cdf09dd682cc7b2eab26abc4f76166fd7e25861ff8de5f2ff84f4273438 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731) 2024-09-17T12:07:01.265 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:07:00 smithi165 ceph-osd[57251]: -- 172.21.15.165:0/3784538718 <== mon.1 v2:172.21.15.165:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55901c0dc1e0 con 0x55901c0d8000 2024-09-17T12:07:01.265 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:07:00 smithi165 bash[57232]: 12ba3cdf09dd682cc7b2eab26abc4f76166fd7e25861ff8de5f2ff84f4273438 2024-09-17T12:07:01.265 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:07:00 smithi165 systemd[1]: Started Ceph osd.6 for a12537de-74ea-11ef-bceb-c7b262605968. 2024-09-17T12:07:01.615 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T12:07:01.615 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (8m) 110s ago 17m 20.4M - 0.27.0 11f11916f8cd 2a2a54402d2a 2024-09-17T12:07:01.615 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (5m) 110s ago 17m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 e545cbf7736d 2024-09-17T12:07:01.615 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (4m) 33s ago 17m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 c1e89e114e4d 2024-09-17T12:07:01.616 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (8m) 110s ago 17m 57.0M - dad864ee21e9 371aaf55379f 2024-09-17T12:07:01.616 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (11m) 110s ago 11m 37.9M - 17.2.0 e1d6a67b021e 6a2a4a998dd1 2024-09-17T12:07:01.616 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (11m) 33s ago 11m 16.9M - 17.2.0 e1d6a67b021e d4c2987b9bac 2024-09-17T12:07:01.616 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:8443,9283,8765 running (7m) 110s ago 18m 520M - 19.3.0-4937-g74984932 6c1fcab85602 ae2f507bd67d 2024-09-17T12:07:01.616 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283,8765 running (6m) 33s ago 16m 457M - 19.3.0-4937-g74984932 6c1fcab85602 53403866652e 2024-09-17T12:07:01.616 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (6m) 110s ago 18m 52.8M 2048M 19.3.0-4937-g74984932 6c1fcab85602 737eec78e0f0 2024-09-17T12:07:01.616 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (5m) 33s ago 16m 44.1M 2048M 19.3.0-4937-g74984932 6c1fcab85602 249eedb4123e 2024-09-17T12:07:01.616 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (11m) 110s ago 11m 279M - 4.0 e1d6a67b021e b8fc749ebb71 2024-09-17T12:07:01.616 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (11m) 33s ago 11m 86.5M - 4.0 e1d6a67b021e 771b27764432 2024-09-17T12:07:01.616 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (8m) 110s ago 17m 11.1M - 1.7.0 72c9c2088986 989b5738cb69 2024-09-17T12:07:01.616 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (7m) 33s ago 16m 12.1M - 1.7.0 72c9c2088986 e1e8ffab3f21 2024-09-17T12:07:01.616 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (4m) 110s ago 16m 338M 1937M 19.3.0-4937-g74984932 6c1fcab85602 20dafd01bd12 2024-09-17T12:07:01.617 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (3m) 110s ago 15m 264M 1937M 19.3.0-4937-g74984932 6c1fcab85602 d39be5014418 2024-09-17T12:07:01.617 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (2m) 110s ago 15m 187M 1937M 19.3.0-4937-g74984932 6c1fcab85602 e13c2ecd369a 2024-09-17T12:07:01.617 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (119s) 110s ago 14m 12.1M 1937M 19.3.0-4937-g74984932 6c1fcab85602 ffd68312d242 2024-09-17T12:07:01.617 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (75s) 33s ago 14m 256M 2704M 19.3.0-4937-g74984932 6c1fcab85602 589de85559fd 2024-09-17T12:07:01.617 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (38s) 33s ago 13m 13.3M 2704M 19.3.0-4937-g74984932 6c1fcab85602 b9ade3d65856 2024-09-17T12:07:01.617 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 starting - - - 2704M 2024-09-17T12:07:01.617 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (13m) 33s ago 13m 744M 2704M 17.2.0 e1d6a67b021e 71b9f2cdb810 2024-09-17T12:07:01.617 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (6m) 110s ago 16m 56.0M - 2.51.0 1d3b7f56885b 61b064ad6609 2024-09-17T12:07:01.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 239551 9.34 MB/sec execute 542 sec latency 36.743 ms 2024-09-17T12:07:02.092 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:01 smithi045 ceph-mon[63880]: from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:07:02.092 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:01 smithi045 ceph-mon[63880]: pgmap v248: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 3.8 MiB/s wr, 336 op/s; 171/1578 objects degraded (10.837%) 2024-09-17T12:07:02.092 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:01 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:02.092 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:01 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:02.093 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:01 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:07:02.093 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:01 smithi045 ceph-mon[63880]: from='client.34414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:07:02.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:01 smithi165 ceph-mon[49373]: from='client.34410 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:07:02.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:01 smithi165 ceph-mon[49373]: pgmap v248: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 3.8 MiB/s wr, 336 op/s; 171/1578 objects degraded (10.837%) 2024-09-17T12:07:02.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:01 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:02.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:01 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:02.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:01 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:07:02.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:01 smithi165 ceph-mon[49373]: from='client.34414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:07:02.331 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:07:02.331 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T12:07:02.331 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:07:02.331 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:07:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T12:07:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:07:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:07:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T12:07:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T12:07:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 6 2024-09-17T12:07:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:07:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T12:07:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T12:07:02.332 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:07:02.333 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T12:07:02.333 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-17T12:07:02.333 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 10 2024-09-17T12:07:02.333 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T12:07:02.333 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:07:02.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 239870 9.34 MB/sec execute 543 sec latency 49.547 ms 2024-09-17T12:07:02.871 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:07:02.871 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T12:07:02.871 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T12:07:02.871 INFO:teuthology.orchestra.run.smithi045.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T12:07:02.871 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [ 2024-09-17T12:07:02.871 INFO:teuthology.orchestra.run.smithi045.stdout: "crash", 2024-09-17T12:07:02.871 INFO:teuthology.orchestra.run.smithi045.stdout: "mon", 2024-09-17T12:07:02.872 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr" 2024-09-17T12:07:02.872 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-17T12:07:02.872 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "12/23 daemons upgraded", 2024-09-17T12:07:02.872 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T12:07:02.872 INFO:teuthology.orchestra.run.smithi045.stdout: "is_paused": false 2024-09-17T12:07:02.872 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:07:02.963 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:07:02 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:07:02.686+0000 7f9674576640 -1 Falling back to public interface 2024-09-17T12:07:03.106 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:02 smithi045 ceph-mon[63880]: from='client.34418 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:07:03.106 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:02 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/2880387596' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:07:03.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:02 smithi165 ceph-mon[49373]: from='client.34418 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:07:03.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:02 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/2880387596' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:07:03.555 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 172/1611 objects degraded (10.677%), 25 pgs degraded 2024-09-17T12:07:03.556 INFO:teuthology.orchestra.run.smithi045.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T12:07:03.556 INFO:teuthology.orchestra.run.smithi045.stdout: osd.6 (root=default,host=smithi165) is down 2024-09-17T12:07:03.556 INFO:teuthology.orchestra.run.smithi045.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 172/1611 objects degraded (10.677%), 25 pgs degraded 2024-09-17T12:07:03.556 INFO:teuthology.orchestra.run.smithi045.stdout: pg 1.0 is active+undersized+degraded, acting [7,0] 2024-09-17T12:07:03.556 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.2 is active+undersized+degraded, acting [5,1] 2024-09-17T12:07:03.556 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.6 is active+undersized+degraded, acting [1,4] 2024-09-17T12:07:03.556 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.7 is active+undersized+degraded, acting [7,2] 2024-09-17T12:07:03.556 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.11 is active+undersized+degraded, acting [4,1] 2024-09-17T12:07:03.556 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.14 is active+undersized+degraded, acting [3,5] 2024-09-17T12:07:03.556 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.16 is active+undersized+degraded, acting [5,2] 2024-09-17T12:07:03.556 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.17 is active+undersized+degraded, acting [5,2] 2024-09-17T12:07:03.557 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.19 is active+undersized+degraded, acting [3,0] 2024-09-17T12:07:03.557 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.1a is active+undersized+degraded, acting [4,7] 2024-09-17T12:07:03.557 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.1b is active+undersized+degraded, acting [3,7] 2024-09-17T12:07:03.557 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.1d is active+undersized+degraded, acting [7,0] 2024-09-17T12:07:03.557 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.0 is active+undersized+degraded, acting [1,2] 2024-09-17T12:07:03.557 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.2 is active+undersized+degraded, acting [3,5] 2024-09-17T12:07:03.557 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.3 is active+undersized+degraded, acting [4,0] 2024-09-17T12:07:03.557 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.a is active+undersized+degraded, acting [4,1] 2024-09-17T12:07:03.557 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.c is active+undersized+degraded, acting [5,3] 2024-09-17T12:07:03.557 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.d is active+undersized+degraded, acting [7,5] 2024-09-17T12:07:03.557 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-09-17T12:07:03.557 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.11 is active+undersized+degraded, acting [7,4] 2024-09-17T12:07:03.557 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.14 is active+undersized+degraded, acting [4,7] 2024-09-17T12:07:03.558 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.1d is active+undersized+degraded, acting [5,4] 2024-09-17T12:07:03.558 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.1e is active+undersized+degraded, acting [3,2] 2024-09-17T12:07:03.558 INFO:teuthology.orchestra.run.smithi045.stdout: pg 4.5 is active+undersized+degraded, acting [4,7] 2024-09-17T12:07:03.558 INFO:teuthology.orchestra.run.smithi045.stdout: pg 4.a is active+undersized+degraded, acting [1,7] 2024-09-17T12:07:03.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 240084 9.33 MB/sec execute 544 sec latency 51.943 ms 2024-09-17T12:07:04.128 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:03 smithi045 ceph-mon[63880]: from='client.34426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:07:04.128 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:03 smithi045 ceph-mon[63880]: pgmap v249: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 4.2 MiB/s wr, 502 op/s; 172/1611 objects degraded (10.677%) 2024-09-17T12:07:04.128 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:03 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 172/1611 objects degraded (10.677%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T12:07:04.128 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:03 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/3108193130' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:07:04.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:03 smithi165 ceph-mon[49373]: from='client.34426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:07:04.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:03 smithi165 ceph-mon[49373]: pgmap v249: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 4.2 MiB/s wr, 502 op/s; 172/1611 objects degraded (10.677%) 2024-09-17T12:07:04.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:03 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 172/1611 objects degraded (10.677%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T12:07:04.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:03 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/3108193130' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:07:04.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 240362 9.33 MB/sec execute 545 sec latency 43.753 ms 2024-09-17T12:07:05.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 240652 9.32 MB/sec execute 546 sec latency 84.401 ms 2024-09-17T12:07:06.151 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:05 smithi045 ceph-mon[63880]: pgmap v250: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.5 MiB/s wr, 584 op/s; 183/1707 objects degraded (10.721%) 2024-09-17T12:07:06.151 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:05 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:06.151 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:05 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:06.151 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:05 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:06.151 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:05 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:06.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:05 smithi165 ceph-mon[49373]: pgmap v250: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.5 MiB/s wr, 584 op/s; 183/1707 objects degraded (10.721%) 2024-09-17T12:07:06.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:05 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:06.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:05 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:06.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:05 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:06.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:05 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:06.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 240939 9.32 MB/sec execute 547 sec latency 65.864 ms 2024-09-17T12:07:07.519 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:07 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:07.519 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:07 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:07:07.519 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:07 smithi045 ceph-mon[63880]: pgmap v251: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 509 op/s; 188/1683 objects degraded (11.171%) 2024-09-17T12:07:07.694 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:07 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:07.694 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:07 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:07:07.694 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:07 smithi165 ceph-mon[49373]: pgmap v251: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 509 op/s; 188/1683 objects degraded (11.171%) 2024-09-17T12:07:07.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 241172 9.31 MB/sec execute 548 sec latency 95.289 ms 2024-09-17T12:07:08.524 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:08 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 188/1683 objects degraded (11.171%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T12:07:08.751 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:08 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 188/1683 objects degraded (11.171%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T12:07:08.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 241453 9.31 MB/sec execute 549 sec latency 41.777 ms 2024-09-17T12:07:09.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 241798 9.30 MB/sec execute 550 sec latency 75.704 ms 2024-09-17T12:07:10.351 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:10 smithi045 ceph-mon[63880]: pgmap v252: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 330 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.1 MiB/s wr, 465 op/s; 179/1584 objects degraded (11.301%) 2024-09-17T12:07:10.351 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:10 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:10.352 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:10 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:10.352 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:10 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:07:10.352 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:10 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:07:10.352 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:10 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:10.352 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:10 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:10.352 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:10 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:07:10.352 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:10 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:07:10.353 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:10 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:07:10.353 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:10 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:07:10.353 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:10 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-17T12:07:10.416 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:10 smithi165 ceph-mon[49373]: pgmap v252: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 330 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.1 MiB/s wr, 465 op/s; 179/1584 objects degraded (11.301%) 2024-09-17T12:07:10.416 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:10 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:10.416 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:10 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:10.416 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:10 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:07:10.416 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:10 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:07:10.416 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:10 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:10.416 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:10 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:10.416 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:10 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:07:10.417 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:10 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:07:10.417 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:10 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:07:10.417 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:10 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:07:10.417 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:10 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-17T12:07:10.765 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:07:10 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:07:10.412+0000 7f9674576640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-17T12:07:10.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 242178 9.30 MB/sec execute 551 sec latency 75.422 ms 2024-09-17T12:07:11.363 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:11 smithi045 ceph-mon[63880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T12:07:11.363 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:11 smithi045 ceph-mon[63880]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-17T12:07:11.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:11 smithi165 ceph-mon[49373]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T12:07:11.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:11 smithi165 ceph-mon[49373]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-17T12:07:11.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 242455 9.30 MB/sec execute 552 sec latency 53.866 ms 2024-09-17T12:07:12.015 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:07:11 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:07:11.723+0000 7f9674576640 -1 osd.6 130 log_to_monitors true 2024-09-17T12:07:12.387 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:12 smithi045 ceph-mon[63880]: pgmap v253: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.6 MiB/s wr, 472 op/s; 178/1590 objects degraded (11.195%) 2024-09-17T12:07:12.387 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:12 smithi045 ceph-mon[63880]: from='osd.6 [v2:172.21.15.165:6816/2377590888,v1:172.21.15.165:6817/2377590888]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-17T12:07:12.387 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:12 smithi045 ceph-mon[63880]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-17T12:07:12.515 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:07:12 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:07:12.165+0000 7f966c320640 -1 osd.6 130 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T12:07:12.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:12 smithi165 ceph-mon[49373]: pgmap v253: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 4.6 MiB/s wr, 472 op/s; 178/1590 objects degraded (11.195%) 2024-09-17T12:07:12.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:12 smithi165 ceph-mon[49373]: from='osd.6 [v2:172.21.15.165:6816/2377590888,v1:172.21.15.165:6817/2377590888]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-17T12:07:12.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:12 smithi165 ceph-mon[49373]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-17T12:07:12.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 242823 9.32 MB/sec execute 553 sec latency 79.866 ms 2024-09-17T12:07:13.415 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:13 smithi045 ceph-mon[63880]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T12:07:13.415 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:13 smithi045 ceph-mon[63880]: osdmap e133: 8 total, 7 up, 8 in 2024-09-17T12:07:13.415 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:13 smithi045 ceph-mon[63880]: from='osd.6 [v2:172.21.15.165:6816/2377590888,v1:172.21.15.165:6817/2377590888]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi165", "root=default"]} : dispatch 2024-09-17T12:07:13.415 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:13 smithi045 ceph-mon[63880]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi165", "root=default"]} : dispatch 2024-09-17T12:07:13.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:13 smithi165 ceph-mon[49373]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T12:07:13.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:13 smithi165 ceph-mon[49373]: osdmap e133: 8 total, 7 up, 8 in 2024-09-17T12:07:13.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:13 smithi165 ceph-mon[49373]: from='osd.6 [v2:172.21.15.165:6816/2377590888,v1:172.21.15.165:6817/2377590888]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi165", "root=default"]} : dispatch 2024-09-17T12:07:13.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:13 smithi165 ceph-mon[49373]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi165", "root=default"]} : dispatch 2024-09-17T12:07:13.774 INFO:teuthology.orchestra.run.smithi045.stdout: 5 243085 9.31 MB/sec execute 554 sec latency 515.053 ms 2024-09-17T12:07:14.440 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:14 smithi045 ceph-mon[63880]: pgmap v255: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.3 MiB/s wr, 373 op/s; 175/1578 objects degraded (11.090%) 2024-09-17T12:07:14.441 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:14 smithi045 ceph-mon[63880]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T12:07:14.441 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:14 smithi045 ceph-mon[63880]: osd.6 [v2:172.21.15.165:6816/2377590888,v1:172.21.15.165:6817/2377590888] boot 2024-09-17T12:07:14.441 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:14 smithi045 ceph-mon[63880]: osdmap e134: 8 total, 8 up, 8 in 2024-09-17T12:07:14.441 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:14 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-17T12:07:14.441 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:14 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 175/1578 objects degraded (11.090%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T12:07:14.441 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:14 smithi045 ceph-mon[63880]: osdmap e135: 8 total, 8 up, 8 in 2024-09-17T12:07:14.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:14 smithi165 ceph-mon[49373]: pgmap v255: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 327 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.3 MiB/s wr, 373 op/s; 175/1578 objects degraded (11.090%) 2024-09-17T12:07:14.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:14 smithi165 ceph-mon[49373]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T12:07:14.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:14 smithi165 ceph-mon[49373]: osd.6 [v2:172.21.15.165:6816/2377590888,v1:172.21.15.165:6817/2377590888] boot 2024-09-17T12:07:14.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:14 smithi165 ceph-mon[49373]: osdmap e134: 8 total, 8 up, 8 in 2024-09-17T12:07:14.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:14 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-17T12:07:14.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:14 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 175/1578 objects degraded (11.090%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T12:07:14.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:14 smithi165 ceph-mon[49373]: osdmap e135: 8 total, 8 up, 8 in 2024-09-17T12:07:14.776 INFO:teuthology.orchestra.run.smithi045.stdout: 5 243264 9.29 MB/sec execute 555 sec latency 1516.544 ms 2024-09-17T12:07:15.465 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:15 smithi045 ceph-mon[63880]: osdmap e136: 8 total, 8 up, 8 in 2024-09-17T12:07:15.465 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:15 smithi045 ceph-mon[63880]: pgmap v259: 97 pgs: 1 remapped+peering, 8 active+undersized, 21 active+undersized+degraded, 67 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 5.4 MiB/s wr, 472 op/s; 140/1650 objects degraded (8.485%); 313 KiB/s, 4 objects/s recovering 2024-09-17T12:07:15.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:15 smithi165 ceph-mon[49373]: osdmap e136: 8 total, 8 up, 8 in 2024-09-17T12:07:15.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:15 smithi165 ceph-mon[49373]: pgmap v259: 97 pgs: 1 remapped+peering, 8 active+undersized, 21 active+undersized+degraded, 67 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 5.4 MiB/s wr, 472 op/s; 140/1650 objects degraded (8.485%); 313 KiB/s, 4 objects/s recovering 2024-09-17T12:07:15.776 INFO:teuthology.orchestra.run.smithi045.stdout: 5 243496 9.29 MB/sec execute 556 sec latency 2015.410 ms 2024-09-17T12:07:16.474 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:16 smithi045 ceph-mon[63880]: osdmap e137: 8 total, 8 up, 8 in 2024-09-17T12:07:16.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:16 smithi165 ceph-mon[49373]: osdmap e137: 8 total, 8 up, 8 in 2024-09-17T12:07:16.776 INFO:teuthology.orchestra.run.smithi045.stdout: 5 243788 9.29 MB/sec execute 557 sec latency 980.319 ms 2024-09-17T12:07:17.511 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:17 smithi045 ceph-mon[63880]: osdmap e138: 8 total, 8 up, 8 in 2024-09-17T12:07:17.511 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:17 smithi045 ceph-mon[63880]: pgmap v262: 97 pgs: 1 remapped+peering, 6 active+undersized, 17 active+undersized+degraded, 73 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 546 B/s rd, 2.8 MiB/s wr, 119 op/s; 109/1662 objects degraded (6.558%); 587 KiB/s, 13 objects/s recovering 2024-09-17T12:07:17.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:17 smithi165 ceph-mon[49373]: osdmap e138: 8 total, 8 up, 8 in 2024-09-17T12:07:17.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:17 smithi165 ceph-mon[49373]: pgmap v262: 97 pgs: 1 remapped+peering, 6 active+undersized, 17 active+undersized+degraded, 73 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 546 B/s rd, 2.8 MiB/s wr, 119 op/s; 109/1662 objects degraded (6.558%); 587 KiB/s, 13 objects/s recovering 2024-09-17T12:07:17.776 INFO:teuthology.orchestra.run.smithi045.stdout: 5 244066 9.28 MB/sec execute 558 sec latency 989.035 ms 2024-09-17T12:07:18.273 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:18 smithi045 ceph-mon[63880]: osdmap e139: 8 total, 8 up, 8 in 2024-09-17T12:07:18.273 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:18 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 109/1662 objects degraded (6.558%), 17 pgs degraded (PG_DEGRADED) 2024-09-17T12:07:18.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:18 smithi165 ceph-mon[49373]: osdmap e139: 8 total, 8 up, 8 in 2024-09-17T12:07:18.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:18 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 109/1662 objects degraded (6.558%), 17 pgs degraded (PG_DEGRADED) 2024-09-17T12:07:18.776 INFO:teuthology.orchestra.run.smithi045.stdout: 5 244386 9.28 MB/sec execute 559 sec latency 45.715 ms 2024-09-17T12:07:19.297 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:19 smithi045 ceph-mon[63880]: pgmap v264: 97 pgs: 3 peering, 94 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.1 MiB/s wr, 415 op/s; 642 KiB/s, 55 keys/s, 42 objects/s recovering 2024-09-17T12:07:19.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:19 smithi165 ceph-mon[49373]: pgmap v264: 97 pgs: 3 peering, 94 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 4.1 MiB/s wr, 415 op/s; 642 KiB/s, 55 keys/s, 42 objects/s recovering 2024-09-17T12:07:19.776 INFO:teuthology.orchestra.run.smithi045.stdout: 5 244752 9.29 MB/sec execute 560 sec latency 42.245 ms 2024-09-17T12:07:20.316 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:20 smithi045 ceph-mon[63880]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 109/1662 objects degraded (6.558%), 17 pgs degraded) 2024-09-17T12:07:20.316 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:20 smithi045 ceph-mon[63880]: Cluster is now healthy 2024-09-17T12:07:20.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:20 smithi165 ceph-mon[49373]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 109/1662 objects degraded (6.558%), 17 pgs degraded) 2024-09-17T12:07:20.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:20 smithi165 ceph-mon[49373]: Cluster is now healthy 2024-09-17T12:07:20.776 INFO:teuthology.orchestra.run.smithi045.stdout: 5 245047 9.28 MB/sec execute 561 sec latency 41.152 ms 2024-09-17T12:07:21.338 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:21 smithi045 ceph-mon[63880]: pgmap v265: 97 pgs: 3 peering, 94 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.6 MiB/s wr, 348 op/s; 504 KiB/s, 43 keys/s, 32 objects/s recovering 2024-09-17T12:07:21.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:21 smithi165 ceph-mon[49373]: pgmap v265: 97 pgs: 3 peering, 94 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.6 MiB/s wr, 348 op/s; 504 KiB/s, 43 keys/s, 32 objects/s recovering 2024-09-17T12:07:21.776 INFO:teuthology.orchestra.run.smithi045.stdout: 5 245291 9.28 MB/sec execute 562 sec latency 64.077 ms 2024-09-17T12:07:22.611 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:22 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:22.611 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:22 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:07:22.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:22 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:22.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:22 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:07:22.776 INFO:teuthology.orchestra.run.smithi045.stdout: 5 245547 9.28 MB/sec execute 563 sec latency 49.890 ms 2024-09-17T12:07:23.629 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:23 smithi045 ceph-mon[63880]: pgmap v266: 97 pgs: 97 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 4.4 MiB/s wr, 426 op/s; 1.4 MiB/s, 33 keys/s, 25 objects/s recovering 2024-09-17T12:07:23.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:23 smithi165 ceph-mon[49373]: pgmap v266: 97 pgs: 97 active+clean; 340 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.6 KiB/s rd, 4.4 MiB/s wr, 426 op/s; 1.4 MiB/s, 33 keys/s, 25 objects/s recovering 2024-09-17T12:07:23.776 INFO:teuthology.orchestra.run.smithi045.stdout: 5 245816 9.27 MB/sec execute 564 sec latency 105.919 ms 2024-09-17T12:07:24.776 INFO:teuthology.orchestra.run.smithi045.stdout: 5 246075 9.26 MB/sec execute 565 sec latency 53.290 ms 2024-09-17T12:07:24.905 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:24 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-17T12:07:25.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:24 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-17T12:07:25.776 INFO:teuthology.orchestra.run.smithi045.stdout: 5 246242 9.25 MB/sec execute 566 sec latency 89.361 ms 2024-09-17T12:07:25.915 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:25 smithi045 ceph-mon[63880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T12:07:25.915 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:25 smithi045 ceph-mon[63880]: Upgrade: osd.7 is safe to restart 2024-09-17T12:07:25.915 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:25 smithi045 ceph-mon[63880]: pgmap v267: 97 pgs: 97 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.9 MiB/s wr, 519 op/s; 1.2 MiB/s, 30 keys/s, 19 objects/s recovering 2024-09-17T12:07:26.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:25 smithi165 ceph-mon[49373]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T12:07:26.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:25 smithi165 ceph-mon[49373]: Upgrade: osd.7 is safe to restart 2024-09-17T12:07:26.016 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:25 smithi165 ceph-mon[49373]: pgmap v267: 97 pgs: 97 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.9 MiB/s wr, 519 op/s; 1.2 MiB/s, 30 keys/s, 19 objects/s recovering 2024-09-17T12:07:26.776 INFO:teuthology.orchestra.run.smithi045.stdout: 5 246493 9.25 MB/sec execute 567 sec latency 74.562 ms 2024-09-17T12:07:27.776 INFO:teuthology.orchestra.run.smithi045.stdout: 5 246701 9.25 MB/sec execute 568 sec latency 76.984 ms 2024-09-17T12:07:28.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:27 smithi045 ceph-mon[63880]: pgmap v268: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.4 MiB/s wr, 462 op/s; 1.0 MiB/s, 26 keys/s, 16 objects/s recovering 2024-09-17T12:07:28.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:27 smithi045 ceph-mon[63880]: Upgrade: Updating osd.7 2024-09-17T12:07:28.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:27 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:28.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:27 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-17T12:07:28.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:27 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:07:28.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:27 smithi045 ceph-mon[63880]: Deploying daemon osd.7 on smithi165 2024-09-17T12:07:28.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:28 smithi165 ceph-mon[49373]: pgmap v268: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.4 MiB/s wr, 462 op/s; 1.0 MiB/s, 26 keys/s, 16 objects/s recovering 2024-09-17T12:07:28.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:28 smithi165 ceph-mon[49373]: Upgrade: Updating osd.7 2024-09-17T12:07:28.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:28 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:28.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:28 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-17T12:07:28.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:28 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:07:28.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:28 smithi165 ceph-mon[49373]: Deploying daemon osd.7 on smithi165 2024-09-17T12:07:28.776 INFO:teuthology.orchestra.run.smithi045.stdout: 5 247035 9.24 MB/sec execute 569 sec latency 74.703 ms 2024-09-17T12:07:29.776 INFO:teuthology.orchestra.run.smithi045.stdout: 5 247510 9.25 MB/sec execute 570 sec latency 41.770 ms 2024-09-17T12:07:30.122 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:30 smithi045 ceph-mon[63880]: pgmap v269: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.7 MiB/s wr, 310 op/s; 817 KiB/s, 5 objects/s recovering 2024-09-17T12:07:30.248 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:30 smithi165 ceph-mon[49373]: pgmap v269: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.7 MiB/s wr, 310 op/s; 817 KiB/s, 5 objects/s recovering 2024-09-17T12:07:30.515 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:30 smithi165 systemd[1]: Stopping Ceph osd.7 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:07:30.776 INFO:teuthology.orchestra.run.smithi045.stdout: 5 247975 9.25 MB/sec execute 571 sec latency 35.400 ms 2024-09-17T12:07:31.015 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:30 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[38891]: 2024-09-17T12:07:30.571+0000 7f49e023b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T12:07:31.015 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:30 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[38891]: 2024-09-17T12:07:30.571+0000 7f49e023b700 -1 osd.7 139 *** Got signal Terminated *** 2024-09-17T12:07:31.015 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:30 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[38891]: 2024-09-17T12:07:30.571+0000 7f49e023b700 -1 osd.7 139 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T12:07:31.133 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:31 smithi045 ceph-mon[63880]: osd.7 marked itself down and dead 2024-09-17T12:07:31.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:31 smithi165 ceph-mon[49373]: osd.7 marked itself down and dead 2024-09-17T12:07:31.776 INFO:teuthology.orchestra.run.smithi045.stdout: 5 248171 9.24 MB/sec execute 572 sec latency 729.811 ms 2024-09-17T12:07:31.857 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:31 smithi165 podman[58829]: 2024-09-17 12:07:31.568276961 +0000 UTC m=+1.119022030 container died 71b9f2cdb810b495ffd20529e2539579cf4a4f63185ca3b5aefc29e34829b669 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7, release=754, CEPH_POINT_RELEASE=-17.2.0, ceph=True, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., name=centos-stream, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD) 2024-09-17T12:07:32.232 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:31 smithi165 podman[58829]: 2024-09-17 12:07:31.946430333 +0000 UTC m=+1.497175395 container cleanup 71b9f2cdb810b495ffd20529e2539579cf4a4f63185ca3b5aefc29e34829b669 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, version=8, distribution-scope=public, io.openshift.expose-services=, GIT_CLEAN=True, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, ceph=True, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8) 2024-09-17T12:07:32.233 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:31 smithi165 bash[58829]: ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7 2024-09-17T12:07:32.233 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:32 smithi165 ceph-mon[49373]: pgmap v270: 97 pgs: 97 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.6 MiB/s wr, 374 op/s; 682 KiB/s, 0 objects/s recovering 2024-09-17T12:07:32.233 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:32 smithi165 ceph-mon[49373]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T12:07:32.233 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:32 smithi165 ceph-mon[49373]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-17T12:07:32.234 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:32 smithi165 ceph-mon[49373]: osdmap e140: 8 total, 7 up, 8 in 2024-09-17T12:07:32.234 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:32 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:32.267 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:32 smithi045 ceph-mon[63880]: pgmap v270: 97 pgs: 97 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.6 MiB/s wr, 374 op/s; 682 KiB/s, 0 objects/s recovering 2024-09-17T12:07:32.267 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:32 smithi045 ceph-mon[63880]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T12:07:32.267 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:32 smithi045 ceph-mon[63880]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-17T12:07:32.267 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:32 smithi045 ceph-mon[63880]: osdmap e140: 8 total, 7 up, 8 in 2024-09-17T12:07:32.267 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:32 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:32.515 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:32 smithi165 podman[58844]: 2024-09-17 12:07:32.231511592 +0000 UTC m=+0.658853042 container remove 71b9f2cdb810b495ffd20529e2539579cf4a4f63185ca3b5aefc29e34829b669 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7, io.openshift.expose-services=, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, vendor=Red Hat, Inc., GIT_CLEAN=True, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, release=754, GIT_BRANCH=HEAD) 2024-09-17T12:07:32.776 INFO:teuthology.orchestra.run.smithi045.stdout: 5 248411 9.24 MB/sec execute 573 sec latency 1005.986 ms 2024-09-17T12:07:32.828 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:32 smithi165 podman[58937]: 2024-09-17 12:07:32.469081295 +0000 UTC m=+0.016863998 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:07:32.828 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:32 smithi165 podman[58937]: 2024-09-17 12:07:32.590915834 +0000 UTC m=+0.138698514 container create adbfafc171ab6d83eade1de66061da115282d0316558c62ea6b0501831181752 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, RELEASE=main-7498493, org.label-schema.license=GPLv2) 2024-09-17T12:07:32.828 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:32 smithi165 podman[58937]: 2024-09-17 12:07:32.827242054 +0000 UTC m=+0.375024748 container init adbfafc171ab6d83eade1de66061da115282d0316558c62ea6b0501831181752 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-7498493, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-17T12:07:33.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:33 smithi165 ceph-mon[49373]: osdmap e141: 8 total, 7 up, 8 in 2024-09-17T12:07:33.265 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:32 smithi165 podman[58937]: 2024-09-17 12:07:32.831281147 +0000 UTC m=+0.379063830 container start adbfafc171ab6d83eade1de66061da115282d0316558c62ea6b0501831181752 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-17T12:07:33.265 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:32 smithi165 podman[58937]: 2024-09-17 12:07:32.927402706 +0000 UTC m=+0.475185403 container attach adbfafc171ab6d83eade1de66061da115282d0316558c62ea6b0501831181752 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T12:07:33.265 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:33 smithi165 podman[58937]: 2024-09-17 12:07:33.050459488 +0000 UTC m=+0.598242176 container died adbfafc171ab6d83eade1de66061da115282d0316558c62ea6b0501831181752 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493) 2024-09-17T12:07:33.275 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:33 smithi045 ceph-mon[63880]: osdmap e141: 8 total, 7 up, 8 in 2024-09-17T12:07:33.776 INFO:teuthology.orchestra.run.smithi045.stdout: 5 248724 9.25 MB/sec execute 574 sec latency 56.253 ms 2024-09-17T12:07:34.015 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:33 smithi165 podman[58960]: 2024-09-17 12:07:33.665781684 +0000 UTC m=+0.608440321 container remove adbfafc171ab6d83eade1de66061da115282d0316558c62ea6b0501831181752 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T12:07:34.015 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:33 smithi165 systemd[1]: ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.7.service: Deactivated successfully. 2024-09-17T12:07:34.015 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:33 smithi165 systemd[1]: Stopped Ceph osd.7 for a12537de-74ea-11ef-bceb-c7b262605968. 2024-09-17T12:07:34.016 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:33 smithi165 systemd[1]: ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.7.service: Consumed 6min 25.799s CPU time. 2024-09-17T12:07:34.016 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:33 smithi165 systemd[1]: Starting Ceph osd.7 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:07:34.163 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T12:07:34.283 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:34 smithi045 ceph-mon[63880]: pgmap v273: 97 pgs: 3 activating+undersized, 1 activating+undersized+degraded, 11 stale+active+clean, 5 active+undersized+degraded, 77 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.0 MiB/s wr, 380 op/s; 33/1575 objects degraded (2.095%) 2024-09-17T12:07:34.283 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:34 smithi045 ceph-mon[63880]: Health check failed: Degraded data redundancy: 33/1575 objects degraded (2.095%), 6 pgs degraded (PG_DEGRADED) 2024-09-17T12:07:34.294 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:34 smithi165 ceph-mon[49373]: pgmap v273: 97 pgs: 3 activating+undersized, 1 activating+undersized+degraded, 11 stale+active+clean, 5 active+undersized+degraded, 77 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.0 MiB/s wr, 380 op/s; 33/1575 objects degraded (2.095%) 2024-09-17T12:07:34.295 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:34 smithi165 ceph-mon[49373]: Health check failed: Degraded data redundancy: 33/1575 objects degraded (2.095%), 6 pgs degraded (PG_DEGRADED) 2024-09-17T12:07:34.295 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:34 smithi165 podman[59061]: 2024-09-17 12:07:34.022846775 +0000 UTC m=+0.020020275 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:07:34.295 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:34 smithi165 podman[59061]: 2024-09-17 12:07:34.150038269 +0000 UTC m=+0.147211755 container create 8c23a939bb36c3faf70d104dfcaf43c63d2691e586d0bb526b39acc3d44c445b (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-activate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=main-7498493, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-17T12:07:34.765 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:34 smithi165 podman[59061]: 2024-09-17 12:07:34.395627365 +0000 UTC m=+0.392800869 container init 8c23a939bb36c3faf70d104dfcaf43c63d2691e586d0bb526b39acc3d44c445b (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-17T12:07:34.765 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:34 smithi165 podman[59061]: 2024-09-17 12:07:34.39925568 +0000 UTC m=+0.396429178 container start 8c23a939bb36c3faf70d104dfcaf43c63d2691e586d0bb526b39acc3d44c445b (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=main-7498493, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T12:07:34.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:34 smithi165 podman[59061]: 2024-09-17 12:07:34.495433066 +0000 UTC m=+0.492606582 container attach 8c23a939bb36c3faf70d104dfcaf43c63d2691e586d0bb526b39acc3d44c445b (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-activate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493) 2024-09-17T12:07:34.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:34 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-activate[59075]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:07:34.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:34 smithi165 bash[59061]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:07:34.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:34 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-activate[59075]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:07:34.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:34 smithi165 bash[59061]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T12:07:34.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 249023 9.25 MB/sec execute 575 sec latency 55.173 ms 2024-09-17T12:07:35.263 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T12:07:35.263 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (8m) 2m ago 18m 20.4M - 0.27.0 11f11916f8cd 2a2a54402d2a 2024-09-17T12:07:35.263 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (5m) 2m ago 18m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 e545cbf7736d 2024-09-17T12:07:35.263 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (5m) 30s ago 17m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 c1e89e114e4d 2024-09-17T12:07:35.263 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (8m) 2m ago 17m 57.0M - dad864ee21e9 371aaf55379f 2024-09-17T12:07:35.264 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (12m) 2m ago 12m 37.9M - 17.2.0 e1d6a67b021e 6a2a4a998dd1 2024-09-17T12:07:35.264 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (12m) 30s ago 12m 17.0M - 17.2.0 e1d6a67b021e d4c2987b9bac 2024-09-17T12:07:35.264 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:8443,9283,8765 running (8m) 2m ago 19m 520M - 19.3.0-4937-g74984932 6c1fcab85602 ae2f507bd67d 2024-09-17T12:07:35.264 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283,8765 running (7m) 30s ago 17m 457M - 19.3.0-4937-g74984932 6c1fcab85602 53403866652e 2024-09-17T12:07:35.264 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (6m) 2m ago 19m 52.8M 2048M 19.3.0-4937-g74984932 6c1fcab85602 737eec78e0f0 2024-09-17T12:07:35.264 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (6m) 30s ago 17m 44.6M 2048M 19.3.0-4937-g74984932 6c1fcab85602 249eedb4123e 2024-09-17T12:07:35.264 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (11m) 2m ago 11m 279M - 4.0 e1d6a67b021e b8fc749ebb71 2024-09-17T12:07:35.264 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (11m) 30s ago 11m 86.5M - 4.0 e1d6a67b021e 771b27764432 2024-09-17T12:07:35.264 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (8m) 2m ago 17m 11.1M - 1.7.0 72c9c2088986 989b5738cb69 2024-09-17T12:07:35.264 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (8m) 30s ago 17m 11.5M - 1.7.0 72c9c2088986 e1e8ffab3f21 2024-09-17T12:07:35.264 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (4m) 2m ago 16m 338M 1937M 19.3.0-4937-g74984932 6c1fcab85602 20dafd01bd12 2024-09-17T12:07:35.265 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (4m) 2m ago 16m 264M 1937M 19.3.0-4937-g74984932 6c1fcab85602 d39be5014418 2024-09-17T12:07:35.265 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (3m) 2m ago 15m 187M 1937M 19.3.0-4937-g74984932 6c1fcab85602 e13c2ecd369a 2024-09-17T12:07:35.265 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (2m) 2m ago 15m 12.1M 1937M 19.3.0-4937-g74984932 6c1fcab85602 ffd68312d242 2024-09-17T12:07:35.265 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (108s) 30s ago 14m 288M 2704M 19.3.0-4937-g74984932 6c1fcab85602 589de85559fd 2024-09-17T12:07:35.265 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (71s) 30s ago 14m 246M 2704M 19.3.0-4937-g74984932 6c1fcab85602 b9ade3d65856 2024-09-17T12:07:35.265 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (34s) 30s ago 14m 11.1M 2704M 19.3.0-4937-g74984932 6c1fcab85602 12ba3cdf09dd 2024-09-17T12:07:35.265 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (13m) 30s ago 13m 796M 2704M 17.2.0 e1d6a67b021e 71b9f2cdb810 2024-09-17T12:07:35.265 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (7m) 2m ago 17m 56.0M - 2.51.0 1d3b7f56885b 61b064ad6609 2024-09-17T12:07:35.290 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:35 smithi045 ceph-mon[63880]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:07:35.352 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:35 smithi165 ceph-mon[49373]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:07:35.352 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:35 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-activate[59075]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T12:07:35.352 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:35 smithi165 bash[59061]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T12:07:35.352 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:35 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-activate[59075]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T12:07:35.352 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:35 smithi165 bash[59061]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T12:07:35.352 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:35 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-activate[59075]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T12:07:35.352 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:35 smithi165 bash[59061]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T12:07:35.713 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:35 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-activate[59075]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T12:07:35.713 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:35 smithi165 bash[59061]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T12:07:35.713 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:35 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-activate[59075]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-17T12:07:35.713 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:35 smithi165 bash[59061]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-17T12:07:35.713 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:35 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-activate[59075]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T12:07:35.714 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:35 smithi165 bash[59061]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T12:07:35.714 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:35 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-activate[59075]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-17T12:07:35.714 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:35 smithi165 bash[59061]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-17T12:07:35.714 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:35 smithi165 podman[59061]: 2024-09-17 12:07:35.385428548 +0000 UTC m=+1.382602048 container died 8c23a939bb36c3faf70d104dfcaf43c63d2691e586d0bb526b39acc3d44c445b (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-activate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, RELEASE=main-7498493, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-17T12:07:35.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 249363 9.24 MB/sec execute 576 sec latency 52.460 ms 2024-09-17T12:07:35.944 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:07:35.944 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T12:07:35.944 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:07:35.944 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:07:35.944 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T12:07:35.944 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:07:35.945 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:07:35.945 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T12:07:35.945 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 7 2024-09-17T12:07:35.945 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:07:35.945 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T12:07:35.945 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T12:07:35.945 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:07:35.945 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T12:07:35.945 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-17T12:07:35.945 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 11 2024-09-17T12:07:35.945 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T12:07:35.945 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:07:36.046 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:36 smithi045 ceph-mon[63880]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:07:36.046 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:36 smithi045 ceph-mon[63880]: pgmap v274: 97 pgs: 5 active+undersized, 3 activating+undersized, 1 activating+undersized+degraded, 2 stale+active+clean, 25 active+undersized+degraded, 61 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.0 MiB/s wr, 449 op/s; 214/1593 objects degraded (13.434%); 0 B/s, 0 objects/s recovering 2024-09-17T12:07:36.046 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:36 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/2648024339' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:07:36.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:36 smithi165 ceph-mon[49373]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:07:36.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:36 smithi165 ceph-mon[49373]: pgmap v274: 97 pgs: 5 active+undersized, 3 activating+undersized, 1 activating+undersized+degraded, 2 stale+active+clean, 25 active+undersized+degraded, 61 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.0 MiB/s wr, 449 op/s; 214/1593 objects degraded (13.434%); 0 B/s, 0 objects/s recovering 2024-09-17T12:07:36.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:36 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/2648024339' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:07:36.266 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:36 smithi165 podman[59215]: 2024-09-17 12:07:36.008763757 +0000 UTC m=+0.612556932 container remove 8c23a939bb36c3faf70d104dfcaf43c63d2691e586d0bb526b39acc3d44c445b (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2) 2024-09-17T12:07:36.266 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:36 smithi165 podman[59275]: 2024-09-17 12:07:36.127071542 +0000 UTC m=+0.016066898 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:07:36.266 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:36 smithi165 podman[59275]: 2024-09-17 12:07:36.251524046 +0000 UTC m=+0.140519388 container create b0f172112f9e76f3ac7a37744e139dfa31edbfe415627209666ab9dcb4e9cc12 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD) 2024-09-17T12:07:36.473 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:07:36.473 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T12:07:36.473 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T12:07:36.473 INFO:teuthology.orchestra.run.smithi045.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T12:07:36.474 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [ 2024-09-17T12:07:36.474 INFO:teuthology.orchestra.run.smithi045.stdout: "crash", 2024-09-17T12:07:36.474 INFO:teuthology.orchestra.run.smithi045.stdout: "mon", 2024-09-17T12:07:36.474 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr" 2024-09-17T12:07:36.474 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-17T12:07:36.474 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "13/23 daemons upgraded", 2024-09-17T12:07:36.474 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T12:07:36.474 INFO:teuthology.orchestra.run.smithi045.stdout: "is_paused": false 2024-09-17T12:07:36.474 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:07:36.765 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:36 smithi165 podman[59275]: 2024-09-17 12:07:36.504892649 +0000 UTC m=+0.393887998 container init b0f172112f9e76f3ac7a37744e139dfa31edbfe415627209666ab9dcb4e9cc12 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=main-7498493, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T12:07:36.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:36 smithi165 podman[59275]: 2024-09-17 12:07:36.508367315 +0000 UTC m=+0.397362657 container start b0f172112f9e76f3ac7a37744e139dfa31edbfe415627209666ab9dcb4e9cc12 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-17T12:07:36.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:36 smithi165 ceph-osd[59294]: -- 172.21.15.165:0/1675257888 <== mon.1 v2:172.21.15.165:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f53f66c1e0 con 0x55f53e877c00 2024-09-17T12:07:36.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:36 smithi165 bash[59275]: b0f172112f9e76f3ac7a37744e139dfa31edbfe415627209666ab9dcb4e9cc12 2024-09-17T12:07:36.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:36 smithi165 systemd[1]: Started Ceph osd.7 for a12537de-74ea-11ef-bceb-c7b262605968. 2024-09-17T12:07:36.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 249581 9.23 MB/sec execute 577 sec latency 75.898 ms 2024-09-17T12:07:37.072 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:37 smithi045 ceph-mon[63880]: from='client.44347 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:07:37.072 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:37 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:37.072 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:37 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:07:37.072 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:37 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:37.072 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:37 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:37.073 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:37 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:07:37.245 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_WARN 1 osds down; all OSDs are running squid or later but require_osd_release < squid; Degraded data redundancy: 214/1593 objects degraded (13.434%), 26 pgs degraded 2024-09-17T12:07:37.245 INFO:teuthology.orchestra.run.smithi045.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T12:07:37.245 INFO:teuthology.orchestra.run.smithi045.stdout: osd.7 (root=default,host=smithi165) is down 2024-09-17T12:07:37.246 INFO:teuthology.orchestra.run.smithi045.stdout:[WRN] OSD_UPGRADE_FINISHED: all OSDs are running squid or later but require_osd_release < squid 2024-09-17T12:07:37.246 INFO:teuthology.orchestra.run.smithi045.stdout: all OSDs are running squid or later but require_osd_release < squid 2024-09-17T12:07:37.246 INFO:teuthology.orchestra.run.smithi045.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 214/1593 objects degraded (13.434%), 26 pgs degraded 2024-09-17T12:07:37.246 INFO:teuthology.orchestra.run.smithi045.stdout: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-09-17T12:07:37.246 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.0 is activating+undersized+degraded, acting [1,0] 2024-09-17T12:07:37.246 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.5 is active+undersized+degraded, acting [0,4] 2024-09-17T12:07:37.246 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.7 is active+undersized+degraded, acting [6,2] 2024-09-17T12:07:37.246 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.8 is active+undersized+degraded, acting [0,1] 2024-09-17T12:07:37.246 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.b is active+undersized+degraded, acting [4,5] 2024-09-17T12:07:37.246 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.e is active+undersized+degraded, acting [2,3] 2024-09-17T12:07:37.246 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.f is active+undersized+degraded, acting [4,0] 2024-09-17T12:07:37.246 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.1a is active+undersized+degraded, acting [6,4] 2024-09-17T12:07:37.246 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.1b is active+undersized+degraded, acting [3,6] 2024-09-17T12:07:37.247 INFO:teuthology.orchestra.run.smithi045.stdout: pg 2.1d is active+undersized+degraded, acting [6,0] 2024-09-17T12:07:37.247 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.7 is active+undersized+degraded, acting [3,0] 2024-09-17T12:07:37.247 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.8 is active+undersized+degraded, acting [3,1] 2024-09-17T12:07:37.247 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.9 is active+undersized+degraded, acting [4,2] 2024-09-17T12:07:37.247 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.e is active+undersized+degraded, acting [4,1] 2024-09-17T12:07:37.247 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.f is active+undersized+degraded, acting [4,0] 2024-09-17T12:07:37.247 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.11 is active+undersized+degraded, acting [4,6] 2024-09-17T12:07:37.247 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.12 is active+undersized+degraded, acting [0,3] 2024-09-17T12:07:37.247 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.13 is active+undersized+degraded, acting [4,2] 2024-09-17T12:07:37.247 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.14 is active+undersized+degraded, acting [4,6] 2024-09-17T12:07:37.247 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.15 is active+undersized+degraded, acting [3,4] 2024-09-17T12:07:37.247 INFO:teuthology.orchestra.run.smithi045.stdout: pg 3.1b is active+undersized+degraded, acting [0,4] 2024-09-17T12:07:37.247 INFO:teuthology.orchestra.run.smithi045.stdout: pg 4.0 is active+undersized+degraded, acting [3,0] 2024-09-17T12:07:37.247 INFO:teuthology.orchestra.run.smithi045.stdout: pg 4.5 is active+undersized+degraded, acting [6,4] 2024-09-17T12:07:37.248 INFO:teuthology.orchestra.run.smithi045.stdout: pg 4.a is active+undersized+degraded, acting [6,1] 2024-09-17T12:07:37.248 INFO:teuthology.orchestra.run.smithi045.stdout: pg 4.14 is active+undersized+degraded, acting [3,1] 2024-09-17T12:07:37.373 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:37 smithi165 ceph-mon[49373]: from='client.44347 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:07:37.373 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:37 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:37.373 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:37 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:07:37.373 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:37 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:37.374 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:37 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:37.374 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:37 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:07:37.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 249761 9.22 MB/sec execute 578 sec latency 71.541 ms 2024-09-17T12:07:38.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:38 smithi045 ceph-mon[63880]: from='client.44355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:07:38.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:38 smithi045 ceph-mon[63880]: pgmap v275: 97 pgs: 7 active+undersized, 3 activating+undersized, 1 activating+undersized+degraded, 31 active+undersized+degraded, 55 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s wr, 395 op/s; 258/1566 objects degraded (16.475%); 0 B/s, 0 objects/s recovering 2024-09-17T12:07:38.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:38 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/288866546' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:07:38.243 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:37 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:07:37.913+0000 7f64d57b8640 -1 Falling back to public interface 2024-09-17T12:07:38.243 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:38 smithi165 ceph-mon[49373]: from='client.44355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:07:38.243 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:38 smithi165 ceph-mon[49373]: pgmap v275: 97 pgs: 7 active+undersized, 3 activating+undersized, 1 activating+undersized+degraded, 31 active+undersized+degraded, 55 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s wr, 395 op/s; 258/1566 objects degraded (16.475%); 0 B/s, 0 objects/s recovering 2024-09-17T12:07:38.243 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:38 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/288866546' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:07:38.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 250046 9.22 MB/sec execute 579 sec latency 43.760 ms 2024-09-17T12:07:39.104 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:39 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 258/1566 objects degraded (16.475%), 32 pgs degraded (PG_DEGRADED) 2024-09-17T12:07:39.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:39 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 258/1566 objects degraded (16.475%), 32 pgs degraded (PG_DEGRADED) 2024-09-17T12:07:39.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 250362 9.22 MB/sec execute 580 sec latency 59.540 ms 2024-09-17T12:07:40.128 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:40 smithi045 ceph-mon[63880]: pgmap v276: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.1 MiB/s wr, 441 op/s; 264/1596 objects degraded (16.541%); 0 B/s, 0 objects/s recovering 2024-09-17T12:07:40.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:40 smithi165 ceph-mon[49373]: pgmap v276: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.1 MiB/s wr, 441 op/s; 264/1596 objects degraded (16.541%); 0 B/s, 0 objects/s recovering 2024-09-17T12:07:40.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 250733 9.23 MB/sec execute 581 sec latency 41.107 ms 2024-09-17T12:07:41.671 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:41 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:41.671 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:41 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:41.671 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:41 smithi045 ceph-mon[63880]: pgmap v277: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.8 MiB/s wr, 463 op/s; 269/1617 objects degraded (16.636%); 0 B/s, 0 objects/s recovering 2024-09-17T12:07:41.671 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:41 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:41.671 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:41 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:41.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:41 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:41.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:41 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:41.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:41 smithi165 ceph-mon[49373]: pgmap v277: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.8 MiB/s wr, 463 op/s; 269/1617 objects degraded (16.636%); 0 B/s, 0 objects/s recovering 2024-09-17T12:07:41.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:41 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:41.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:41 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:41.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 251031 9.23 MB/sec execute 582 sec latency 56.549 ms 2024-09-17T12:07:42.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 251362 9.22 MB/sec execute 583 sec latency 63.667 ms 2024-09-17T12:07:43.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 251763 9.22 MB/sec execute 584 sec latency 42.361 ms 2024-09-17T12:07:44.214 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:44 smithi045 ceph-mon[63880]: pgmap v278: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.8 MiB/s wr, 505 op/s; 269/1638 objects degraded (16.422%); 0 B/s, 0 objects/s recovering 2024-09-17T12:07:44.214 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:44 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 269/1638 objects degraded (16.422%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T12:07:44.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:44 smithi165 ceph-mon[49373]: pgmap v278: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.8 MiB/s wr, 505 op/s; 269/1638 objects degraded (16.422%); 0 B/s, 0 objects/s recovering 2024-09-17T12:07:44.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:44 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 269/1638 objects degraded (16.422%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T12:07:44.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 252139 9.23 MB/sec execute 585 sec latency 38.152 ms 2024-09-17T12:07:45.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 252495 9.23 MB/sec execute 586 sec latency 66.242 ms 2024-09-17T12:07:46.227 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: pgmap v279: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 337 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.0 MiB/s wr, 421 op/s; 263/1659 objects degraded (15.853%); 0 B/s, 0 objects/s recovering 2024-09-17T12:07:46.228 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:46.228 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:46.228 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:07:46.228 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:07:46.228 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:46.228 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:46.228 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:07:46.228 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:07:46.228 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:07:46.228 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:07:46.228 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:07:46.228 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:46.229 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-17T12:07:46.229 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-17T12:07:46.229 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-17T12:07:46.229 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-17T12:07:46.229 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-17T12:07:46.229 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-17T12:07:46.229 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-17T12:07:46.229 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-17T12:07:46.229 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-17T12:07:46.229 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-17T12:07:46.229 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-17T12:07:46.229 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-17T12:07:46.229 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-17T12:07:46.229 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-17T12:07:46.230 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-17T12:07:46.230 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-17T12:07:46.230 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:45 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-17T12:07:46.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: pgmap v279: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 337 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.0 MiB/s wr, 421 op/s; 263/1659 objects degraded (15.853%); 0 B/s, 0 objects/s recovering 2024-09-17T12:07:46.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:46.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:46.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:07:46.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:07:46.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:46.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:46.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:07:46.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:07:46.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:07:46.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:07:46.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:07:46.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:46.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-17T12:07:46.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-17T12:07:46.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-17T12:07:46.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-17T12:07:46.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-17T12:07:46.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-17T12:07:46.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-17T12:07:46.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-17T12:07:46.268 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-17T12:07:46.268 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-17T12:07:46.268 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-17T12:07:46.268 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-17T12:07:46.268 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-17T12:07:46.268 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-17T12:07:46.268 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-17T12:07:46.268 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-17T12:07:46.268 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:45 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-17T12:07:46.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 253044 9.24 MB/sec execute 587 sec latency 39.949 ms 2024-09-17T12:07:46.980 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:46 smithi045 ceph-mon[63880]: Upgrade: Setting container_image for all osd 2024-09-17T12:07:46.980 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:46 smithi045 ceph-mon[63880]: Upgrade: Setting require_osd_release to 19 squid 2024-09-17T12:07:46.980 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:46 smithi045 ceph-mon[63880]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-17T12:07:46.980 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:46 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-17T12:07:46.980 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:46 smithi045 ceph-mon[63880]: osdmap e142: 8 total, 7 up, 8 in 2024-09-17T12:07:46.980 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:46 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi045.olwaas"]} : dispatch 2024-09-17T12:07:47.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:46 smithi165 ceph-mon[49373]: Upgrade: Setting container_image for all osd 2024-09-17T12:07:47.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:46 smithi165 ceph-mon[49373]: Upgrade: Setting require_osd_release to 19 squid 2024-09-17T12:07:47.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:46 smithi165 ceph-mon[49373]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-17T12:07:47.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:46 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-17T12:07:47.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:46 smithi165 ceph-mon[49373]: osdmap e142: 8 total, 7 up, 8 in 2024-09-17T12:07:47.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:46 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi045.olwaas"]} : dispatch 2024-09-17T12:07:47.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 253605 9.25 MB/sec execute 588 sec latency 30.232 ms 2024-09-17T12:07:47.986 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:47 smithi045 ceph-mon[63880]: Upgrade: It appears safe to stop mds.foofs.smithi045.olwaas 2024-09-17T12:07:47.986 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:47 smithi045 ceph-mon[63880]: pgmap v281: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 337 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 446 op/s; 262/1683 objects degraded (15.567%) 2024-09-17T12:07:48.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:47 smithi165 ceph-mon[49373]: Upgrade: It appears safe to stop mds.foofs.smithi045.olwaas 2024-09-17T12:07:48.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:47 smithi165 ceph-mon[49373]: pgmap v281: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 337 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 446 op/s; 262/1683 objects degraded (15.567%) 2024-09-17T12:07:48.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 254071 9.25 MB/sec execute 589 sec latency 32.994 ms 2024-09-17T12:07:48.991 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:48 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 262/1683 objects degraded (15.567%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T12:07:49.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:48 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 262/1683 objects degraded (15.567%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T12:07:49.266 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:48 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:07:48.870+0000 7f64d57b8640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-17T12:07:49.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 254396 9.25 MB/sec execute 590 sec latency 65.386 ms 2024-09-17T12:07:50.002 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:49 smithi045 ceph-mon[63880]: pgmap v282: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.0 MiB/s wr, 621 op/s; 274/1749 objects degraded (15.666%) 2024-09-17T12:07:50.002 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:49 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:50.002 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:49 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi045.olwaas", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T12:07:50.002 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:49 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:07:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:49 smithi165 ceph-mon[49373]: pgmap v282: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 346 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.0 MiB/s wr, 621 op/s; 274/1749 objects degraded (15.666%) 2024-09-17T12:07:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:49 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:49 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi045.olwaas", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T12:07:50.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:49 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:07:50.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 254779 9.26 MB/sec execute 591 sec latency 53.140 ms 2024-09-17T12:07:50.994 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:50 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:07:50.553+0000 7f64d57b8640 -1 osd.7 139 log_to_monitors true 2024-09-17T12:07:51.015 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:50 smithi045 ceph-mon[63880]: Upgrade: Updating mds.foofs.smithi045.olwaas 2024-09-17T12:07:51.015 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:50 smithi045 ceph-mon[63880]: Deploying daemon mds.foofs.smithi045.olwaas on smithi045 2024-09-17T12:07:51.015 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:50 smithi045 ceph-mon[63880]: from='osd.7 [v2:172.21.15.165:6824/2402997451,v1:172.21.15.165:6825/2402997451]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-17T12:07:51.015 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:50 smithi045 ceph-mon[63880]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-17T12:07:51.265 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:51 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:07:51.007+0000 7f64cd562640 -1 osd.7 139 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T12:07:51.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:50 smithi165 ceph-mon[49373]: Upgrade: Updating mds.foofs.smithi045.olwaas 2024-09-17T12:07:51.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:50 smithi165 ceph-mon[49373]: Deploying daemon mds.foofs.smithi045.olwaas on smithi045 2024-09-17T12:07:51.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:50 smithi165 ceph-mon[49373]: from='osd.7 [v2:172.21.15.165:6824/2402997451,v1:172.21.15.165:6825/2402997451]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-17T12:07:51.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:50 smithi165 ceph-mon[49373]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-17T12:07:51.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 255144 9.25 MB/sec execute 592 sec latency 40.131 ms 2024-09-17T12:07:52.062 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:52 smithi045 ceph-mon[63880]: pgmap v283: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.0 MiB/s wr, 632 op/s; 270/1728 objects degraded (15.625%) 2024-09-17T12:07:52.062 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:52 smithi045 ceph-mon[63880]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T12:07:52.062 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:52 smithi045 ceph-mon[63880]: osdmap e143: 8 total, 7 up, 8 in 2024-09-17T12:07:52.062 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:52 smithi045 ceph-mon[63880]: from='osd.7 [v2:172.21.15.165:6824/2402997451,v1:172.21.15.165:6825/2402997451]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi165", "root=default"]} : dispatch 2024-09-17T12:07:52.062 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:52 smithi045 ceph-mon[63880]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi165", "root=default"]} : dispatch 2024-09-17T12:07:52.062 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:52 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:52.062 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:52 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:07:52.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:52 smithi165 ceph-mon[49373]: pgmap v283: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.0 MiB/s wr, 632 op/s; 270/1728 objects degraded (15.625%) 2024-09-17T12:07:52.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:52 smithi165 ceph-mon[49373]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T12:07:52.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:52 smithi165 ceph-mon[49373]: osdmap e143: 8 total, 7 up, 8 in 2024-09-17T12:07:52.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:52 smithi165 ceph-mon[49373]: from='osd.7 [v2:172.21.15.165:6824/2402997451,v1:172.21.15.165:6825/2402997451]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi165", "root=default"]} : dispatch 2024-09-17T12:07:52.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:52 smithi165 ceph-mon[49373]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi165", "root=default"]} : dispatch 2024-09-17T12:07:52.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:52 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:07:52.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:52 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:07:52.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 255312 9.24 MB/sec execute 593 sec latency 744.933 ms 2024-09-17T12:07:53.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:52 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045[63876]: 2024-09-17T12:07:52.781+0000 7f87e59fb640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T12:07:53.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:53 smithi045 ceph-mon[63880]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T12:07:53.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:53 smithi045 ceph-mon[63880]: osd.7 [v2:172.21.15.165:6824/2402997451,v1:172.21.15.165:6825/2402997451] boot 2024-09-17T12:07:53.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:53 smithi045 ceph-mon[63880]: osdmap e144: 8 total, 8 up, 8 in 2024-09-17T12:07:53.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:53 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-17T12:07:53.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:53 smithi045 ceph-mon[63880]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-17T12:07:53.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:53 smithi045 ceph-mon[63880]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T12:07:53.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:53 smithi045 ceph-mon[63880]: osdmap e145: 8 total, 8 up, 8 in 2024-09-17T12:07:53.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:53 smithi045 ceph-mon[63880]: Standby daemon mds.foofs.smithi165.czpmgd assigned to filesystem foofs as rank 0 2024-09-17T12:07:53.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:53 smithi045 ceph-mon[63880]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-17T12:07:53.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:53 smithi045 ceph-mon[63880]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T12:07:53.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:53 smithi045 ceph-mon[63880]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-17T12:07:53.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:53 smithi045 ceph-mon[63880]: fsmap foofs:1/1 {0=foofs.smithi165.czpmgd=up:replay} 2024-09-17T12:07:53.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:53 smithi165 ceph-mon[49373]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T12:07:53.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:53 smithi165 ceph-mon[49373]: osd.7 [v2:172.21.15.165:6824/2402997451,v1:172.21.15.165:6825/2402997451] boot 2024-09-17T12:07:53.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:53 smithi165 ceph-mon[49373]: osdmap e144: 8 total, 8 up, 8 in 2024-09-17T12:07:53.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:53 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-17T12:07:53.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:53 smithi165 ceph-mon[49373]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-17T12:07:53.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:53 smithi165 ceph-mon[49373]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T12:07:53.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:53 smithi165 ceph-mon[49373]: osdmap e145: 8 total, 8 up, 8 in 2024-09-17T12:07:53.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:53 smithi165 ceph-mon[49373]: Standby daemon mds.foofs.smithi165.czpmgd assigned to filesystem foofs as rank 0 2024-09-17T12:07:53.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:53 smithi165 ceph-mon[49373]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-17T12:07:53.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:53 smithi165 ceph-mon[49373]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T12:07:53.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:53 smithi165 ceph-mon[49373]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-17T12:07:53.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:53 smithi165 ceph-mon[49373]: fsmap foofs:1/1 {0=foofs.smithi165.czpmgd=up:replay} 2024-09-17T12:07:53.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 255312 9.22 MB/sec execute 594 sec latency 1744.989 ms 2024-09-17T12:07:54.265 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:07:53 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:07:53.850+0000 7f96562da640 -1 bdev(0x55901c132c00 /var/lib/ceph/osd/ceph-6/block) aio_submit retries 3 2024-09-17T12:07:54.265 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:07:53 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:07:53.850+0000 7f64b5518640 -1 bdev(0x55f53f6c2c00 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 2 2024-09-17T12:07:54.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:54 smithi165 ceph-mon[49373]: pgmap v287: 97 pgs: 7 peering, 7 active+undersized, 28 active+undersized+degraded, 55 active+clean; 340 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 6.9 MiB/s wr, 778 op/s; 247/1728 objects degraded (14.294%) 2024-09-17T12:07:54.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:54 smithi165 ceph-mon[49373]: Health check update: Degraded data redundancy: 270/1728 objects degraded (15.625%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T12:07:54.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:54 smithi165 ceph-mon[49373]: osdmap e146: 8 total, 8 up, 8 in 2024-09-17T12:07:54.266 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:07:53 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:07:53.856+0000 7f1bad34d640 -1 bdev(0x55ad0f302c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 5 2024-09-17T12:07:54.266 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:07:53 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:07:53.856+0000 7f1ba7b42640 -1 bdev(0x55ad0f302c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 5 2024-09-17T12:07:54.328 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:54 smithi045 ceph-mon[63880]: pgmap v287: 97 pgs: 7 peering, 7 active+undersized, 28 active+undersized+degraded, 55 active+clean; 340 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 6.9 MiB/s wr, 778 op/s; 247/1728 objects degraded (14.294%) 2024-09-17T12:07:54.328 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:54 smithi045 ceph-mon[63880]: Health check update: Degraded data redundancy: 270/1728 objects degraded (15.625%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T12:07:54.328 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:54 smithi045 ceph-mon[63880]: osdmap e146: 8 total, 8 up, 8 in 2024-09-17T12:07:54.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 255314 9.21 MB/sec execute 595 sec latency 1926.533 ms 2024-09-17T12:07:55.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 255314 9.19 MB/sec execute 596 sec latency 2926.577 ms 2024-09-17T12:07:56.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:55 smithi045 ceph-mon[63880]: osdmap e147: 8 total, 8 up, 8 in 2024-09-17T12:07:56.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:55 smithi045 ceph-mon[63880]: pgmap v290: 97 pgs: 2 active+undersized+remapped, 9 remapped+peering, 7 peering, 2 active+undersized, 13 active+undersized+degraded, 64 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 2.9 MiB/s wr, 169 op/s; 82/1698 objects degraded (4.829%); 1.3 MiB/s, 12 objects/s recovering 2024-09-17T12:07:56.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:55 smithi045 ceph-mon[63880]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-17T12:07:56.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:55 smithi045 ceph-mon[63880]: reconnect by client.24369 172.21.15.165:0/1603532166 after 0 2024-09-17T12:07:56.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:55 smithi045 ceph-mon[63880]: reconnect by client.14548 172.21.15.45:0/1025805077 after 0 2024-09-17T12:07:56.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:55 smithi045 ceph-mon[63880]: mds.? [v2:172.21.15.45:6834/1484615878,v1:172.21.15.45:6835/1484615878] up:boot 2024-09-17T12:07:56.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:55 smithi045 ceph-mon[63880]: mds.? [v2:172.21.15.165:6832/1936605703,v1:172.21.15.165:6833/1936605703] up:reconnect 2024-09-17T12:07:56.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:55 smithi045 ceph-mon[63880]: fsmap foofs:1/1 {0=foofs.smithi165.czpmgd=up:reconnect} 1 up:standby 2024-09-17T12:07:56.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:55 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mds metadata", "who": "foofs.smithi045.olwaas"} : dispatch 2024-09-17T12:07:56.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:55 smithi165 ceph-mon[49373]: osdmap e147: 8 total, 8 up, 8 in 2024-09-17T12:07:56.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:55 smithi165 ceph-mon[49373]: pgmap v290: 97 pgs: 2 active+undersized+remapped, 9 remapped+peering, 7 peering, 2 active+undersized, 13 active+undersized+degraded, 64 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 2.9 MiB/s wr, 169 op/s; 82/1698 objects degraded (4.829%); 1.3 MiB/s, 12 objects/s recovering 2024-09-17T12:07:56.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:55 smithi165 ceph-mon[49373]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-17T12:07:56.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:55 smithi165 ceph-mon[49373]: reconnect by client.24369 172.21.15.165:0/1603532166 after 0 2024-09-17T12:07:56.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:55 smithi165 ceph-mon[49373]: reconnect by client.14548 172.21.15.45:0/1025805077 after 0 2024-09-17T12:07:56.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:55 smithi165 ceph-mon[49373]: mds.? [v2:172.21.15.45:6834/1484615878,v1:172.21.15.45:6835/1484615878] up:boot 2024-09-17T12:07:56.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:55 smithi165 ceph-mon[49373]: mds.? [v2:172.21.15.165:6832/1936605703,v1:172.21.15.165:6833/1936605703] up:reconnect 2024-09-17T12:07:56.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:55 smithi165 ceph-mon[49373]: fsmap foofs:1/1 {0=foofs.smithi165.czpmgd=up:reconnect} 1 up:standby 2024-09-17T12:07:56.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:55 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mds metadata", "who": "foofs.smithi045.olwaas"} : dispatch 2024-09-17T12:07:56.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 255314 9.18 MB/sec execute 597 sec latency 3926.625 ms 2024-09-17T12:07:57.059 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:56 smithi045 ceph-mon[63880]: osdmap e148: 8 total, 8 up, 8 in 2024-09-17T12:07:57.059 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:56 smithi045 ceph-mon[63880]: mds.? [v2:172.21.15.165:6832/1936605703,v1:172.21.15.165:6833/1936605703] up:rejoin 2024-09-17T12:07:57.059 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:56 smithi045 ceph-mon[63880]: fsmap foofs:1/1 {0=foofs.smithi165.czpmgd=up:rejoin} 1 up:standby 2024-09-17T12:07:57.059 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:56 smithi045 ceph-mon[63880]: daemon mds.foofs.smithi165.czpmgd is now active in filesystem foofs as rank 0 2024-09-17T12:07:57.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:56 smithi165 ceph-mon[49373]: osdmap e148: 8 total, 8 up, 8 in 2024-09-17T12:07:57.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:56 smithi165 ceph-mon[49373]: mds.? [v2:172.21.15.165:6832/1936605703,v1:172.21.15.165:6833/1936605703] up:rejoin 2024-09-17T12:07:57.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:56 smithi165 ceph-mon[49373]: fsmap foofs:1/1 {0=foofs.smithi165.czpmgd=up:rejoin} 1 up:standby 2024-09-17T12:07:57.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:56 smithi165 ceph-mon[49373]: daemon mds.foofs.smithi165.czpmgd is now active in filesystem foofs as rank 0 2024-09-17T12:07:57.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 255531 9.18 MB/sec execute 598 sec latency 4211.375 ms 2024-09-17T12:07:57.821 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:57 smithi045 ceph-mon[63880]: pgmap v292: 97 pgs: 1 active+undersized+remapped, 6 remapped+peering, 7 peering, 83 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 3.6 MiB/s wr, 264 op/s; 2.9 MiB/s, 49 objects/s recovering 2024-09-17T12:07:57.821 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:57 smithi045 ceph-mon[63880]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-17T12:07:57.821 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:57 smithi045 ceph-mon[63880]: mds.? [v2:172.21.15.165:6832/1936605703,v1:172.21.15.165:6833/1936605703] up:active 2024-09-17T12:07:57.821 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:57 smithi045 ceph-mon[63880]: fsmap foofs:1 {0=foofs.smithi165.czpmgd=up:active} 1 up:standby 2024-09-17T12:07:58.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:57 smithi165 ceph-mon[49373]: pgmap v292: 97 pgs: 1 active+undersized+remapped, 6 remapped+peering, 7 peering, 83 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 3.6 MiB/s wr, 264 op/s; 2.9 MiB/s, 49 objects/s recovering 2024-09-17T12:07:58.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:57 smithi165 ceph-mon[49373]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-17T12:07:58.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:57 smithi165 ceph-mon[49373]: mds.? [v2:172.21.15.165:6832/1936605703,v1:172.21.15.165:6833/1936605703] up:active 2024-09-17T12:07:58.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:57 smithi165 ceph-mon[49373]: fsmap foofs:1 {0=foofs.smithi165.czpmgd=up:active} 1 up:standby 2024-09-17T12:07:58.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 255832 9.18 MB/sec execute 599 sec latency 40.370 ms 2024-09-17T12:07:58.843 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:58 smithi045 ceph-mon[63880]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 82/1698 objects degraded (4.829%), 13 pgs degraded) 2024-09-17T12:07:58.843 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:58 smithi045 ceph-mon[63880]: Cluster is now healthy 2024-09-17T12:07:59.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:58 smithi165 ceph-mon[49373]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 82/1698 objects degraded (4.829%), 13 pgs degraded) 2024-09-17T12:07:59.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:58 smithi165 ceph-mon[49373]: Cluster is now healthy 2024-09-17T12:07:59.777 INFO:teuthology.orchestra.run.smithi045.stdout: 5 cleanup 600 sec 2024-09-17T12:07:59.860 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:07:59 smithi045 ceph-mon[63880]: pgmap v293: 97 pgs: 1 active+undersized+remapped, 96 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 3.3 MiB/s wr, 226 op/s; 3.1 MiB/s, 57 objects/s recovering 2024-09-17T12:08:00.042 INFO:teuthology.orchestra.run.smithi045.stdout: 0 cleanup 600 sec 2024-09-17T12:08:00.042 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T12:08:00.042 INFO:teuthology.orchestra.run.smithi045.stdout: Operation Count AvgLat MaxLat 2024-09-17T12:08:00.042 INFO:teuthology.orchestra.run.smithi045.stdout: ---------------------------------------- 2024-09-17T12:08:00.042 INFO:teuthology.orchestra.run.smithi045.stdout: NTCreateX 176386 4.841 2180.419 2024-09-17T12:08:00.042 INFO:teuthology.orchestra.run.smithi045.stdout: Close 129456 0.143 10.745 2024-09-17T12:08:00.042 INFO:teuthology.orchestra.run.smithi045.stdout: Rename 7493 8.726 1032.090 2024-09-17T12:08:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: Unlink 35678 7.077 2177.196 2024-09-17T12:08:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: Qpathinfo 159874 1.329 4211.369 2024-09-17T12:08:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: Qfileinfo 27897 0.007 1.035 2024-09-17T12:08:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: Qfsinfo 29418 0.685 69.414 2024-09-17T12:08:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: Sfileinfo 14366 13.266 3212.820 2024-09-17T12:08:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: Find 61889 1.028 17.644 2024-09-17T12:08:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: WriteX 87194 15.095 3224.087 2024-09-17T12:08:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: ReadX 277565 0.010 3.231 2024-09-17T12:08:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: LockX 578 0.417 4.691 2024-09-17T12:08:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: UnlockX 578 0.363 4.095 2024-09-17T12:08:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: Flush 12364 0.003 0.086 2024-09-17T12:08:00.043 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T12:08:00.043 INFO:teuthology.orchestra.run.smithi045.stdout:Throughput 9.17887 MB/sec 5 clients 5 procs max_latency=4211.375 ms 2024-09-17T12:08:00.045 DEBUG:teuthology.orchestra.run.smithi045:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-09-17T12:08:00.083 DEBUG:teuthology.orchestra.run.smithi045:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-09-17T12:08:00.209 DEBUG:teuthology.orchestra.run.smithi045:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-09-17T12:08:00.236 INFO:teuthology.orchestra.run.smithi045.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-17T12:08:00.237 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T12:08:00.237 INFO:teuthology.orchestra.run.smithi045.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-09-17T12:08:00.237 INFO:teuthology.orchestra.run.smithi045.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-17T12:08:00.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:07:59 smithi165 ceph-mon[49373]: pgmap v293: 97 pgs: 1 active+undersized+remapped, 96 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 3.3 MiB/s wr, 226 op/s; 3.1 MiB/s, 57 objects/s recovering 2024-09-17T12:08:00.337 INFO:teuthology.orchestra.run.smithi045.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-17T12:08:00.337 INFO:teuthology.orchestra.run.smithi045.stdout:releasing clients 2024-09-17T12:08:02.206 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:02 smithi045 ceph-mon[63880]: pgmap v294: 97 pgs: 97 active+clean; 342 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 2.7 MiB/s wr, 225 op/s; 2.5 MiB/s, 45 objects/s recovering 2024-09-17T12:08:02.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:02 smithi165 ceph-mon[49373]: pgmap v294: 97 pgs: 97 active+clean; 342 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 2.7 MiB/s wr, 225 op/s; 2.5 MiB/s, 45 objects/s recovering 2024-09-17T12:08:02.337 INFO:teuthology.orchestra.run.smithi045.stdout: 5 380 20.81 MB/sec execute 1 sec latency 60.931 ms 2024-09-17T12:08:03.337 INFO:teuthology.orchestra.run.smithi045.stdout: 5 541 23.72 MB/sec execute 2 sec latency 117.132 ms 2024-09-17T12:08:04.291 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:04 smithi045 ceph-mon[63880]: pgmap v295: 97 pgs: 97 active+clean; 352 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 5.4 MiB/s wr, 333 op/s; 1.7 MiB/s, 36 objects/s recovering 2024-09-17T12:08:04.337 INFO:teuthology.orchestra.run.smithi045.stdout: 5 822 20.18 MB/sec execute 3 sec latency 198.301 ms 2024-09-17T12:08:04.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:04 smithi165 ceph-mon[49373]: pgmap v295: 97 pgs: 97 active+clean; 352 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 5.4 MiB/s wr, 333 op/s; 1.7 MiB/s, 36 objects/s recovering 2024-09-17T12:08:05.338 INFO:teuthology.orchestra.run.smithi045.stdout: 5 1148 16.99 MB/sec execute 4 sec latency 79.845 ms 2024-09-17T12:08:06.082 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:06 smithi045 ceph-mon[63880]: pgmap v296: 97 pgs: 97 active+clean; 338 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 8.9 MiB/s wr, 334 op/s; 1.4 MiB/s, 29 objects/s recovering 2024-09-17T12:08:06.082 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:06 smithi045 ceph-mon[63880]: daemon mds.foofs.smithi045.olwaas restarted 2024-09-17T12:08:06.082 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:06 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:06.082 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:06 smithi045 ceph-mon[63880]: osdmap e149: 8 total, 8 up, 8 in 2024-09-17T12:08:06.082 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:06 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:06.082 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:06 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:08:06.338 INFO:teuthology.orchestra.run.smithi045.stdout: 5 cleanup 5 sec 2024-09-17T12:08:06.510 INFO:teuthology.orchestra.run.smithi045.stdout:[1546] unlink './clients/client1/~dmtmp/PARADOX/.nfs000001000000b2350000003c' failed - Device or resource busy 2024-09-17T12:08:06.510 INFO:teuthology.orchestra.run.smithi045.stdout:[1546] unlink './clients/client1/~dmtmp/PARADOX/.nfs000001000000b24000000027' failed - Device or resource busy 2024-09-17T12:08:06.515 INFO:teuthology.orchestra.run.smithi045.stdout:[1534] unlink './clients/client2/~dmtmp/PARADOX/.nfs000001000000b23700000044' failed - Device or resource busy 2024-09-17T12:08:06.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:06 smithi165 ceph-mon[49373]: pgmap v296: 97 pgs: 97 active+clean; 338 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 8.9 MiB/s wr, 334 op/s; 1.4 MiB/s, 29 objects/s recovering 2024-09-17T12:08:06.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:06 smithi165 ceph-mon[49373]: daemon mds.foofs.smithi045.olwaas restarted 2024-09-17T12:08:06.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:06 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:06.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:06 smithi165 ceph-mon[49373]: osdmap e149: 8 total, 8 up, 8 in 2024-09-17T12:08:06.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:06 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:06.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:06 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:08:06.516 INFO:teuthology.orchestra.run.smithi045.stdout:[1534] unlink './clients/client2/~dmtmp/PARADOX/.nfs000001000000b24600000047' failed - Device or resource busy 2024-09-17T12:08:06.516 INFO:teuthology.orchestra.run.smithi045.stdout:[1546] unlink './clients/client1/~dmtmp/PARADOX/.nfs000001000000b3310000002c' failed - Device or resource busy 2024-09-17T12:08:06.516 INFO:teuthology.orchestra.run.smithi045.stdout:[1534] unlink './clients/client2/~dmtmp/PARADOX/.nfs000001000000b33300000034' failed - Device or resource busy 2024-09-17T12:08:06.516 INFO:teuthology.orchestra.run.smithi045.stdout:[1534] unlink './clients/client2/~dmtmp/PARADOX/.nfs000001000000b33000000041' failed - Device or resource busy 2024-09-17T12:08:06.517 INFO:teuthology.orchestra.run.smithi045.stdout:[1534] unlink './clients/client2/~dmtmp/PARADOX/.nfs000001000000b23c00000028' failed - Device or resource busy 2024-09-17T12:08:06.520 INFO:teuthology.orchestra.run.smithi045.stdout:[1534] unlink './clients/client2/~dmtmp/PARADOX/.nfs000001000000b24100000030' failed - Device or resource busy 2024-09-17T12:08:06.520 INFO:teuthology.orchestra.run.smithi045.stdout:[1546] unlink './clients/client1/~dmtmp/PARADOX/.nfs000001000000b2450000003f' failed - Device or resource busy 2024-09-17T12:08:06.596 INFO:teuthology.orchestra.run.smithi045.stdout: 0 cleanup 5 sec 2024-09-17T12:08:06.596 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T12:08:06.596 INFO:teuthology.orchestra.run.smithi045.stdout: Operation Count AvgLat MaxLat 2024-09-17T12:08:06.596 INFO:teuthology.orchestra.run.smithi045.stdout: ---------------------------------------- 2024-09-17T12:08:06.596 INFO:teuthology.orchestra.run.smithi045.stdout: NTCreateX 1038 6.459 60.923 2024-09-17T12:08:06.596 INFO:teuthology.orchestra.run.smithi045.stdout: Close 876 6.504 198.293 2024-09-17T12:08:06.596 INFO:teuthology.orchestra.run.smithi045.stdout: Rename 50 11.733 62.740 2024-09-17T12:08:06.596 INFO:teuthology.orchestra.run.smithi045.stdout: Unlink 103 4.297 20.760 2024-09-17T12:08:06.596 INFO:teuthology.orchestra.run.smithi045.stdout: Qpathinfo 684 1.736 33.043 2024-09-17T12:08:06.597 INFO:teuthology.orchestra.run.smithi045.stdout: Qfileinfo 377 15.379 52.432 2024-09-17T12:08:06.597 INFO:teuthology.orchestra.run.smithi045.stdout: Qfsinfo 165 0.494 2.600 2024-09-17T12:08:06.597 INFO:teuthology.orchestra.run.smithi045.stdout: Sfileinfo 170 13.695 51.209 2024-09-17T12:08:06.597 INFO:teuthology.orchestra.run.smithi045.stdout: Find 287 0.972 3.555 2024-09-17T12:08:06.597 INFO:teuthology.orchestra.run.smithi045.stdout: WriteX 1641 0.024 0.085 2024-09-17T12:08:06.597 INFO:teuthology.orchestra.run.smithi045.stdout: ReadX 1233 0.015 0.643 2024-09-17T12:08:06.597 INFO:teuthology.orchestra.run.smithi045.stdout: Flush 128 14.538 52.794 2024-09-17T12:08:06.597 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T12:08:06.597 INFO:teuthology.orchestra.run.smithi045.stdout:Throughput 16.9947 MB/sec 5 clients 5 procs max_latency=198.301 ms 2024-09-17T12:08:06.599 DEBUG:teuthology.parallel:result is None 2024-09-17T12:08:07.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:07 smithi045 ceph-mon[63880]: mds.? [v2:172.21.15.45:6834/2587321174,v1:172.21.15.45:6835/2587321174] up:boot 2024-09-17T12:08:07.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:07 smithi045 ceph-mon[63880]: fsmap foofs:1 {0=foofs.smithi165.czpmgd=up:active} 1 up:standby 2024-09-17T12:08:07.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:07 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mds metadata", "who": "foofs.smithi045.olwaas"} : dispatch 2024-09-17T12:08:07.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:07 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:07.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:07 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:08:07.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:07 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:07.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:07 smithi165 ceph-mon[49373]: mds.? [v2:172.21.15.45:6834/2587321174,v1:172.21.15.45:6835/2587321174] up:boot 2024-09-17T12:08:07.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:07 smithi165 ceph-mon[49373]: fsmap foofs:1 {0=foofs.smithi165.czpmgd=up:active} 1 up:standby 2024-09-17T12:08:07.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:07 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mds metadata", "who": "foofs.smithi045.olwaas"} : dispatch 2024-09-17T12:08:07.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:07 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:07.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:07 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:08:07.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:07 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:07.659 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T12:08:08.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:08 smithi045 ceph-mon[63880]: pgmap v298: 97 pgs: 97 active+clean; 321 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 12 MiB/s wr, 461 op/s; 663 KiB/s, 13 objects/s recovering 2024-09-17T12:08:08.430 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T12:08:08.430 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (9m) 2m ago 18m 20.4M - 0.27.0 11f11916f8cd 2a2a54402d2a 2024-09-17T12:08:08.430 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (6m) 2m ago 18m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 e545cbf7736d 2024-09-17T12:08:08.430 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (6m) 27s ago 18m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 c1e89e114e4d 2024-09-17T12:08:08.430 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (9m) 2m ago 18m 57.0M - dad864ee21e9 371aaf55379f 2024-09-17T12:08:08.430 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 starting - - - - 2024-09-17T12:08:08.430 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (12m) 27s ago 12m 17.1M - 17.2.0 e1d6a67b021e d4c2987b9bac 2024-09-17T12:08:08.430 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:8443,9283,8765 running (8m) 2m ago 19m 520M - 19.3.0-4937-g74984932 6c1fcab85602 ae2f507bd67d 2024-09-17T12:08:08.431 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283,8765 running (7m) 27s ago 18m 457M - 19.3.0-4937-g74984932 6c1fcab85602 53403866652e 2024-09-17T12:08:08.431 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (7m) 2m ago 20m 52.8M 2048M 19.3.0-4937-g74984932 6c1fcab85602 737eec78e0f0 2024-09-17T12:08:08.431 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (6m) 27s ago 17m 47.1M 2048M 19.3.0-4937-g74984932 6c1fcab85602 249eedb4123e 2024-09-17T12:08:08.431 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (12m) 2m ago 12m 279M - 4.0 e1d6a67b021e b8fc749ebb71 2024-09-17T12:08:08.431 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (12m) 27s ago 12m 86.5M - 4.0 e1d6a67b021e 771b27764432 2024-09-17T12:08:08.431 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (9m) 2m ago 18m 11.1M - 1.7.0 72c9c2088986 989b5738cb69 2024-09-17T12:08:08.431 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (8m) 27s ago 17m 11.2M - 1.7.0 72c9c2088986 e1e8ffab3f21 2024-09-17T12:08:08.431 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (5m) 2m ago 17m 338M 1937M 19.3.0-4937-g74984932 6c1fcab85602 20dafd01bd12 2024-09-17T12:08:08.431 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (4m) 2m ago 16m 264M 1937M 19.3.0-4937-g74984932 6c1fcab85602 d39be5014418 2024-09-17T12:08:08.431 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (3m) 2m ago 16m 187M 1937M 19.3.0-4937-g74984932 6c1fcab85602 e13c2ecd369a 2024-09-17T12:08:08.431 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (3m) 2m ago 15m 12.1M 1937M 19.3.0-4937-g74984932 6c1fcab85602 ffd68312d242 2024-09-17T12:08:08.431 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (2m) 27s ago 15m 322M 2704M 19.3.0-4937-g74984932 6c1fcab85602 589de85559fd 2024-09-17T12:08:08.431 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (105s) 27s ago 15m 278M 2704M 19.3.0-4937-g74984932 6c1fcab85602 b9ade3d65856 2024-09-17T12:08:08.432 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (67s) 27s ago 14m 237M 2704M 19.3.0-4937-g74984932 6c1fcab85602 12ba3cdf09dd 2024-09-17T12:08:08.432 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (32s) 27s ago 14m 11.6M 2704M 19.3.0-4937-g74984932 6c1fcab85602 b0f172112f9e 2024-09-17T12:08:08.432 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (7m) 2m ago 17m 56.0M - 2.51.0 1d3b7f56885b 61b064ad6609 2024-09-17T12:08:08.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:08 smithi165 ceph-mon[49373]: pgmap v298: 97 pgs: 97 active+clean; 321 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 12 MiB/s wr, 461 op/s; 663 KiB/s, 13 objects/s recovering 2024-09-17T12:08:08.918 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:08:08.918 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T12:08:08.918 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:08:08.918 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:08:08.919 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T12:08:08.919 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:08:08.919 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:08:08.919 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T12:08:08.919 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 8 2024-09-17T12:08:08.919 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:08:08.919 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T12:08:08.919 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T12:08:08.919 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 1 2024-09-17T12:08:08.919 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:08:08.919 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T12:08:08.919 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T12:08:08.919 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 13 2024-09-17T12:08:08.920 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T12:08:08.920 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:08:09.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:09 smithi045 ceph-mon[63880]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:08:09.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:09 smithi045 ceph-mon[63880]: from='client.34474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:08:09.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:09 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/1676003611' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:09.344 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:08:09.345 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T12:08:09.345 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T12:08:09.345 INFO:teuthology.orchestra.run.smithi045.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T12:08:09.345 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [ 2024-09-17T12:08:09.345 INFO:teuthology.orchestra.run.smithi045.stdout: "crash", 2024-09-17T12:08:09.345 INFO:teuthology.orchestra.run.smithi045.stdout: "osd", 2024-09-17T12:08:09.345 INFO:teuthology.orchestra.run.smithi045.stdout: "mon", 2024-09-17T12:08:09.345 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr" 2024-09-17T12:08:09.345 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-17T12:08:09.345 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "14/23 daemons upgraded", 2024-09-17T12:08:09.345 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "Currently upgrading mds daemons", 2024-09-17T12:08:09.345 INFO:teuthology.orchestra.run.smithi045.stdout: "is_paused": false 2024-09-17T12:08:09.346 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:08:09.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:09 smithi165 ceph-mon[49373]: from='client.34470 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:08:09.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:09 smithi165 ceph-mon[49373]: from='client.34474 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:08:09.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:09 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/1676003611' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:09.786 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_OK 2024-09-17T12:08:10.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:10 smithi045 ceph-mon[63880]: from='client.34478 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:08:10.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:10 smithi045 ceph-mon[63880]: pgmap v299: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 862 KiB/s rd, 13 MiB/s wr, 510 op/s 2024-09-17T12:08:10.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:10 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/4087916076' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:08:10.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:10 smithi165 ceph-mon[49373]: from='client.34478 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:08:10.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:10 smithi165 ceph-mon[49373]: pgmap v299: 97 pgs: 97 active+clean; 328 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 862 KiB/s rd, 13 MiB/s wr, 510 op/s 2024-09-17T12:08:10.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:10 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/4087916076' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:08:11.322 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:11 smithi045 ceph-mon[63880]: from='client.34486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:08:11.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:11 smithi165 ceph-mon[49373]: from='client.34486 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:08:12.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:12 smithi045 ceph-mon[63880]: pgmap v300: 97 pgs: 97 active+clean; 314 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 12 MiB/s wr, 445 op/s 2024-09-17T12:08:12.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:12 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:12.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:12 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:12.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:12 smithi165 ceph-mon[49373]: pgmap v300: 97 pgs: 97 active+clean; 314 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 12 MiB/s wr, 445 op/s 2024-09-17T12:08:12.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:12 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:12.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:12 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:13.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:13 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:13.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:13 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:13.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:13 smithi045 ceph-mon[63880]: pgmap v301: 97 pgs: 97 active+clean; 272 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 9.6 MiB/s wr, 350 op/s 2024-09-17T12:08:13.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:13 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:13.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:13 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:13.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:13 smithi165 ceph-mon[49373]: pgmap v301: 97 pgs: 97 active+clean; 272 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 9.6 MiB/s wr, 350 op/s 2024-09-17T12:08:16.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:16 smithi045 ceph-mon[63880]: pgmap v302: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.2 MiB/s wr, 305 op/s 2024-09-17T12:08:16.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:16 smithi165 ceph-mon[49373]: pgmap v302: 97 pgs: 97 active+clean; 226 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.2 MiB/s wr, 305 op/s 2024-09-17T12:08:17.129 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:17 smithi165 ceph-mon[49373]: Detected new or changed devices on smithi045 2024-09-17T12:08:17.130 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:17 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:17.130 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:17 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:17.130 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:17 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:08:17.130 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:17 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:08:17.130 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:17 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:17.130 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:17 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:17.130 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:17 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:08:17.130 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:17 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:17.130 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:17 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:17.130 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:17 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:17.130 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:17 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:17.130 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:17 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi165.czpmgd"]} : dispatch 2024-09-17T12:08:17.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:17 smithi045 ceph-mon[63880]: Detected new or changed devices on smithi045 2024-09-17T12:08:17.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:17 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:17.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:17 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:17.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:17 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:08:17.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:17 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:08:17.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:17 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:17.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:17 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:17.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:17 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:08:17.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:17 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:17.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:17 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:17.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:17 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:17.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:17 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:17.341 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:17 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi165.czpmgd"]} : dispatch 2024-09-17T12:08:18.330 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:18 smithi165 ceph-mon[49373]: Upgrade: It appears safe to stop mds.foofs.smithi165.czpmgd 2024-09-17T12:08:18.330 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:18 smithi165 ceph-mon[49373]: pgmap v303: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.1 MiB/s wr, 138 op/s 2024-09-17T12:08:18.330 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:18 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:18.330 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:18 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi165.czpmgd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T12:08:18.330 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:18 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:08:18.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:18 smithi045 ceph-mon[63880]: Upgrade: It appears safe to stop mds.foofs.smithi165.czpmgd 2024-09-17T12:08:18.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:18 smithi045 ceph-mon[63880]: pgmap v303: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.1 MiB/s wr, 138 op/s 2024-09-17T12:08:18.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:18 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:18.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:18 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi165.czpmgd", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-17T12:08:18.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:18 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:08:19.293 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:19 smithi165 ceph-mon[49373]: Upgrade: Updating mds.foofs.smithi165.czpmgd 2024-09-17T12:08:19.293 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:19 smithi165 ceph-mon[49373]: Deploying daemon mds.foofs.smithi165.czpmgd on smithi165 2024-09-17T12:08:19.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:19 smithi045 ceph-mon[63880]: Upgrade: Updating mds.foofs.smithi165.czpmgd 2024-09-17T12:08:19.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:19 smithi045 ceph-mon[63880]: Deploying daemon mds.foofs.smithi165.czpmgd on smithi165 2024-09-17T12:08:20.265 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:08:19 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:08:19.865+0000 7f64b7d1d640 -1 bdev(0x55f53f6c2c00 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 1 2024-09-17T12:08:20.265 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:08:19 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:08:19.868+0000 7f1ba7b42640 -1 bdev(0x55ad0f302c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 3 2024-09-17T12:08:20.266 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:08:19 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:08:19.870+0000 7f1bac34b640 -1 bdev(0x55ad0f302c00 /var/lib/ceph/osd/ceph-5/block) aio_submit retries 3 2024-09-17T12:08:20.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:20 smithi165 ceph-mon[49373]: pgmap v304: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.3 KiB/s rd, 1.7 MiB/s wr, 97 op/s 2024-09-17T12:08:20.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:20 smithi165 ceph-mon[49373]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-17T12:08:20.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:20 smithi165 ceph-mon[49373]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T12:08:20.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:20 smithi165 ceph-mon[49373]: osdmap e150: 8 total, 8 up, 8 in 2024-09-17T12:08:20.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:20 smithi165 ceph-mon[49373]: Standby daemon mds.foofs.smithi045.olwaas assigned to filesystem foofs as rank 0 2024-09-17T12:08:20.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:20 smithi165 ceph-mon[49373]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-17T12:08:20.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:20 smithi165 ceph-mon[49373]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T12:08:20.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:20 smithi165 ceph-mon[49373]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-17T12:08:20.267 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:20 smithi165 ceph-mon[49373]: fsmap foofs:1/1 {0=foofs.smithi045.olwaas=up:replay} 2024-09-17T12:08:20.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:19 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045[63876]: 2024-09-17T12:08:19.846+0000 7f87e59fb640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T12:08:20.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:20 smithi045 ceph-mon[63880]: pgmap v304: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.3 KiB/s rd, 1.7 MiB/s wr, 97 op/s 2024-09-17T12:08:20.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:20 smithi045 ceph-mon[63880]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-17T12:08:20.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:20 smithi045 ceph-mon[63880]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T12:08:20.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:20 smithi045 ceph-mon[63880]: osdmap e150: 8 total, 8 up, 8 in 2024-09-17T12:08:20.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:20 smithi045 ceph-mon[63880]: Standby daemon mds.foofs.smithi045.olwaas assigned to filesystem foofs as rank 0 2024-09-17T12:08:20.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:20 smithi045 ceph-mon[63880]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-17T12:08:20.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:20 smithi045 ceph-mon[63880]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T12:08:20.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:20 smithi045 ceph-mon[63880]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-17T12:08:20.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:20 smithi045 ceph-mon[63880]: fsmap foofs:1/1 {0=foofs.smithi045.olwaas=up:replay} 2024-09-17T12:08:22.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:22 smithi045 ceph-mon[63880]: pgmap v306: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 73 KiB/s wr, 36 op/s 2024-09-17T12:08:22.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:22 smithi045 ceph-mon[63880]: reconnect by client.14548 172.21.15.45:0/1025805077 after 0.000999986 2024-09-17T12:08:22.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:22 smithi045 ceph-mon[63880]: reconnect by client.24369 172.21.15.165:0/1603532166 after 0.000999986 2024-09-17T12:08:22.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:22 smithi045 ceph-mon[63880]: mds.? [v2:172.21.15.45:6834/2587321174,v1:172.21.15.45:6835/2587321174] up:reconnect 2024-09-17T12:08:22.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:22 smithi045 ceph-mon[63880]: fsmap foofs:1/1 {0=foofs.smithi045.olwaas=up:reconnect} 2024-09-17T12:08:22.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:22 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:22.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:22 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:08:22.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:22 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mds metadata", "who": "foofs.smithi165.czpmgd"} : dispatch 2024-09-17T12:08:22.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:22 smithi165 ceph-mon[49373]: pgmap v306: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 73 KiB/s wr, 36 op/s 2024-09-17T12:08:22.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:22 smithi165 ceph-mon[49373]: reconnect by client.14548 172.21.15.45:0/1025805077 after 0.000999986 2024-09-17T12:08:22.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:22 smithi165 ceph-mon[49373]: reconnect by client.24369 172.21.15.165:0/1603532166 after 0.000999986 2024-09-17T12:08:22.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:22 smithi165 ceph-mon[49373]: mds.? [v2:172.21.15.45:6834/2587321174,v1:172.21.15.45:6835/2587321174] up:reconnect 2024-09-17T12:08:22.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:22 smithi165 ceph-mon[49373]: fsmap foofs:1/1 {0=foofs.smithi045.olwaas=up:reconnect} 2024-09-17T12:08:22.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:22 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:22.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:22 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:08:22.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:22 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mds metadata", "who": "foofs.smithi165.czpmgd"} : dispatch 2024-09-17T12:08:23.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:23 smithi045 ceph-mon[63880]: mds.? [v2:172.21.15.45:6834/2587321174,v1:172.21.15.45:6835/2587321174] up:rejoin 2024-09-17T12:08:23.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:23 smithi045 ceph-mon[63880]: fsmap foofs:1/1 {0=foofs.smithi045.olwaas=up:rejoin} 2024-09-17T12:08:23.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:23 smithi045 ceph-mon[63880]: daemon mds.foofs.smithi045.olwaas is now active in filesystem foofs as rank 0 2024-09-17T12:08:23.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:23 smithi165 ceph-mon[49373]: mds.? [v2:172.21.15.45:6834/2587321174,v1:172.21.15.45:6835/2587321174] up:rejoin 2024-09-17T12:08:23.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:23 smithi165 ceph-mon[49373]: fsmap foofs:1/1 {0=foofs.smithi045.olwaas=up:rejoin} 2024-09-17T12:08:23.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:23 smithi165 ceph-mon[49373]: daemon mds.foofs.smithi045.olwaas is now active in filesystem foofs as rank 0 2024-09-17T12:08:24.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:24 smithi165 ceph-mon[49373]: pgmap v307: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 62 KiB/s wr, 22 op/s 2024-09-17T12:08:24.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:24 smithi165 ceph-mon[49373]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-17T12:08:24.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:24 smithi165 ceph-mon[49373]: mds.? [v2:172.21.15.45:6834/2587321174,v1:172.21.15.45:6835/2587321174] up:active 2024-09-17T12:08:24.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:24 smithi165 ceph-mon[49373]: fsmap foofs:1 {0=foofs.smithi045.olwaas=up:active} 2024-09-17T12:08:24.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:24 smithi045 ceph-mon[63880]: pgmap v307: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 16 MiB/s rd, 62 KiB/s wr, 22 op/s 2024-09-17T12:08:24.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:24 smithi045 ceph-mon[63880]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-17T12:08:24.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:24 smithi045 ceph-mon[63880]: mds.? [v2:172.21.15.45:6834/2587321174,v1:172.21.15.45:6835/2587321174] up:active 2024-09-17T12:08:24.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:24 smithi045 ceph-mon[63880]: fsmap foofs:1 {0=foofs.smithi045.olwaas=up:active} 2024-09-17T12:08:26.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:26 smithi045 ceph-mon[63880]: pgmap v308: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 4.5 KiB/s wr, 10 op/s 2024-09-17T12:08:26.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:26 smithi165 ceph-mon[49373]: pgmap v308: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 4.5 KiB/s wr, 10 op/s 2024-09-17T12:08:28.377 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:28 smithi165 ceph-mon[49373]: pgmap v309: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 4.5 KiB/s wr, 10 op/s 2024-09-17T12:08:28.469 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:28 smithi045 ceph-mon[63880]: pgmap v309: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 4.5 KiB/s wr, 10 op/s 2024-09-17T12:08:30.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:29 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:30.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:29 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:30.089 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:29 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:08:30.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:29 smithi045 ceph-mon[63880]: pgmap v310: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 9.1 KiB/s wr, 14 op/s 2024-09-17T12:08:30.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:29 smithi045 ceph-mon[63880]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-17T12:08:30.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:29 smithi045 ceph-mon[63880]: Cluster is now healthy 2024-09-17T12:08:30.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:29 smithi045 ceph-mon[63880]: mds.? [v2:172.21.15.165:6832/2910920311,v1:172.21.15.165:6833/2910920311] up:boot 2024-09-17T12:08:30.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:29 smithi045 ceph-mon[63880]: fsmap foofs:1 {0=foofs.smithi045.olwaas=up:active} 1 up:standby 2024-09-17T12:08:30.090 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:29 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mds metadata", "who": "foofs.smithi165.czpmgd"} : dispatch 2024-09-17T12:08:30.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:29 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:30.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:29 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:30.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:29 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:08:30.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:29 smithi165 ceph-mon[49373]: pgmap v310: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 22 MiB/s rd, 9.1 KiB/s wr, 14 op/s 2024-09-17T12:08:30.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:29 smithi165 ceph-mon[49373]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-17T12:08:30.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:29 smithi165 ceph-mon[49373]: Cluster is now healthy 2024-09-17T12:08:30.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:29 smithi165 ceph-mon[49373]: mds.? [v2:172.21.15.165:6832/2910920311,v1:172.21.15.165:6833/2910920311] up:boot 2024-09-17T12:08:30.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:29 smithi165 ceph-mon[49373]: fsmap foofs:1 {0=foofs.smithi045.olwaas=up:active} 1 up:standby 2024-09-17T12:08:30.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:29 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "mds metadata", "who": "foofs.smithi165.czpmgd"} : dispatch 2024-09-17T12:08:32.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:32 smithi045 ceph-mon[63880]: pgmap v311: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 8.2 KiB/s wr, 13 op/s 2024-09-17T12:08:32.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:32 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:32.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:32 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:32.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:32 smithi165 ceph-mon[49373]: pgmap v311: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 8.2 KiB/s wr, 13 op/s 2024-09-17T12:08:32.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:32 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:32.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:32 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:33.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:33 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:33.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:33 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:33.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:33 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:33.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:33 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:34.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:34 smithi165 ceph-mon[49373]: pgmap v312: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 7.6 KiB/s wr, 11 op/s 2024-09-17T12:08:34.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:34 smithi045 ceph-mon[63880]: pgmap v312: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 7.6 KiB/s wr, 11 op/s 2024-09-17T12:08:36.271 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: pgmap v313: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 7.5 KiB/s wr, 8 op/s 2024-09-17T12:08:36.271 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: Detected new or changed devices on smithi165 2024-09-17T12:08:36.271 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:36.271 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:36.271 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:08:36.271 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:08:36.271 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:36.271 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:36.271 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:08:36.271 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:36.272 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:36.272 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:36.272 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:36.272 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:36.272 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:36.272 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi045.olwaas"} : dispatch 2024-09-17T12:08:36.272 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi045.olwaas"}]': finished 2024-09-17T12:08:36.272 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi165.czpmgd"} : dispatch 2024-09-17T12:08:36.272 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi165.czpmgd"}]': finished 2024-09-17T12:08:36.272 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:36.272 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:36.272 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:36.272 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:36.272 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:36.273 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:36.273 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:36.273 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:36.273 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:36 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:36.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: pgmap v313: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 7.5 KiB/s wr, 8 op/s 2024-09-17T12:08:36.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: Detected new or changed devices on smithi165 2024-09-17T12:08:36.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:36.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:36.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:08:36.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:08:36.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:36.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:36.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:08:36.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:36.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:36.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:36.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:36.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:36.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:36.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi045.olwaas"} : dispatch 2024-09-17T12:08:36.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi045.olwaas"}]': finished 2024-09-17T12:08:36.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi165.czpmgd"} : dispatch 2024-09-17T12:08:36.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi165.czpmgd"}]': finished 2024-09-17T12:08:36.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:36.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:36.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:36.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:36.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:36.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:36.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:36.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:36.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:36 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:37.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:37 smithi045 ceph-mon[63880]: Upgrade: Setting container_image for all mds 2024-09-17T12:08:37.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:37 smithi045 ceph-mon[63880]: Upgrade: Setting container_image for all rgw 2024-09-17T12:08:37.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:37 smithi045 ceph-mon[63880]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T12:08:37.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:37 smithi045 ceph-mon[63880]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T12:08:37.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:37 smithi045 ceph-mon[63880]: Upgrade: Setting container_image for all iscsi 2024-09-17T12:08:37.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:37 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:37.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:37 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:08:37.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:37 smithi165 ceph-mon[49373]: Upgrade: Setting container_image for all mds 2024-09-17T12:08:37.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:37 smithi165 ceph-mon[49373]: Upgrade: Setting container_image for all rgw 2024-09-17T12:08:37.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:37 smithi165 ceph-mon[49373]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T12:08:37.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:37 smithi165 ceph-mon[49373]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T12:08:37.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:37 smithi165 ceph-mon[49373]: Upgrade: Setting container_image for all iscsi 2024-09-17T12:08:37.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:37 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:37.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:37 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:08:38.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:38 smithi045 ceph-mon[63880]: pgmap v314: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 7.1 KiB/s wr, 4 op/s 2024-09-17T12:08:38.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:38 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:38.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:38 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi045.qochkw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-17T12:08:38.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:38 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-17T12:08:38.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:38 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T12:08:38.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:38 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:08:38.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:38 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-17T12:08:38.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:38 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T12:08:38.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:38 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi045.qochkw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-17T12:08:38.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:38 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:08:38.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:38 smithi165 ceph-mon[49373]: pgmap v314: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 7.1 KiB/s wr, 4 op/s 2024-09-17T12:08:38.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:38 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:38.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:38 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi045.qochkw", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-17T12:08:38.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:38 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-17T12:08:38.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:38 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T12:08:38.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:38 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:08:38.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:38 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-17T12:08:38.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:38 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T12:08:38.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:38 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi045.qochkw-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-17T12:08:38.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:38 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:08:39.422 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:39 smithi045 ceph-mon[63880]: Upgrade: Updating nfs.foo.0.0.smithi045.qochkw (1/2) 2024-09-17T12:08:39.422 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:39 smithi045 ceph-mon[63880]: Creating key for client.nfs.foo.0.0.smithi045.qochkw 2024-09-17T12:08:39.422 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:39 smithi045 ceph-mon[63880]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-17T12:08:39.422 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:39 smithi045 ceph-mon[63880]: Rados config object exists: conf-nfs.foo 2024-09-17T12:08:39.422 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:39 smithi045 ceph-mon[63880]: Creating key for client.nfs.foo.0.0.smithi045.qochkw-rgw 2024-09-17T12:08:39.422 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:39 smithi045 ceph-mon[63880]: Bind address in nfs.foo.0.0.smithi045.qochkw's ganesha conf is defaulting to empty 2024-09-17T12:08:39.422 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:39 smithi045 ceph-mon[63880]: Deploying daemon nfs.foo.0.0.smithi045.qochkw on smithi045 2024-09-17T12:08:39.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:39 smithi165 ceph-mon[49373]: Upgrade: Updating nfs.foo.0.0.smithi045.qochkw (1/2) 2024-09-17T12:08:39.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:39 smithi165 ceph-mon[49373]: Creating key for client.nfs.foo.0.0.smithi045.qochkw 2024-09-17T12:08:39.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:39 smithi165 ceph-mon[49373]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-17T12:08:39.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:39 smithi165 ceph-mon[49373]: Rados config object exists: conf-nfs.foo 2024-09-17T12:08:39.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:39 smithi165 ceph-mon[49373]: Creating key for client.nfs.foo.0.0.smithi045.qochkw-rgw 2024-09-17T12:08:39.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:39 smithi165 ceph-mon[49373]: Bind address in nfs.foo.0.0.smithi045.qochkw's ganesha conf is defaulting to empty 2024-09-17T12:08:39.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:39 smithi165 ceph-mon[49373]: Deploying daemon nfs.foo.0.0.smithi045.qochkw on smithi045 2024-09-17T12:08:40.177 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T12:08:40.399 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:40 smithi045 ceph-mon[63880]: pgmap v315: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.5 KiB/s rd, 7.2 KiB/s wr, 4 op/s 2024-09-17T12:08:40.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:40 smithi165 ceph-mon[49373]: pgmap v315: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.5 KiB/s rd, 7.2 KiB/s wr, 4 op/s 2024-09-17T12:08:40.976 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T12:08:40.976 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (10m) 29s ago 19m 20.1M - 0.27.0 11f11916f8cd 2a2a54402d2a 2024-09-17T12:08:40.976 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (6m) 29s ago 19m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 e545cbf7736d 2024-09-17T12:08:40.976 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (6m) 9s ago 18m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 c1e89e114e4d 2024-09-17T12:08:40.976 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (9m) 29s ago 19m 58.6M - dad864ee21e9 371aaf55379f 2024-09-17T12:08:40.976 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (35s) 29s ago 13m 13.1M - 19.3.0-4937-g74984932 6c1fcab85602 1db3f808b6e6 2024-09-17T12:08:40.977 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (12s) 9s ago 13m 11.8M - 19.3.0-4937-g74984932 6c1fcab85602 b13106022d43 2024-09-17T12:08:40.977 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:8443,9283,8765 running (9m) 29s ago 20m 525M - 19.3.0-4937-g74984932 6c1fcab85602 ae2f507bd67d 2024-09-17T12:08:40.977 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283,8765 running (8m) 9s ago 18m 458M - 19.3.0-4937-g74984932 6c1fcab85602 53403866652e 2024-09-17T12:08:40.977 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (7m) 29s ago 20m 61.7M 2048M 19.3.0-4937-g74984932 6c1fcab85602 737eec78e0f0 2024-09-17T12:08:40.977 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (7m) 9s ago 18m 49.0M 2048M 19.3.0-4937-g74984932 6c1fcab85602 249eedb4123e 2024-09-17T12:08:40.977 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (12m) 29s ago 12m 379M - 4.0 e1d6a67b021e b8fc749ebb71 2024-09-17T12:08:40.977 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (12m) 9s ago 12m 86.6M - 4.0 e1d6a67b021e 771b27764432 2024-09-17T12:08:40.977 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (9m) 29s ago 18m 11.8M - 1.7.0 72c9c2088986 989b5738cb69 2024-09-17T12:08:40.977 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (9m) 9s ago 18m 10.7M - 1.7.0 72c9c2088986 e1e8ffab3f21 2024-09-17T12:08:40.977 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (6m) 29s ago 17m 399M 1937M 19.3.0-4937-g74984932 6c1fcab85602 20dafd01bd12 2024-09-17T12:08:40.977 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (5m) 29s ago 17m 319M 1937M 19.3.0-4937-g74984932 6c1fcab85602 d39be5014418 2024-09-17T12:08:40.977 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (4m) 29s ago 16m 364M 1937M 19.3.0-4937-g74984932 6c1fcab85602 e13c2ecd369a 2024-09-17T12:08:40.977 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (3m) 29s ago 16m 380M 1937M 19.3.0-4937-g74984932 6c1fcab85602 ffd68312d242 2024-09-17T12:08:40.977 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (2m) 9s ago 15m 348M 2704M 19.3.0-4937-g74984932 6c1fcab85602 589de85559fd 2024-09-17T12:08:40.978 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (2m) 9s ago 15m 332M 2704M 19.3.0-4937-g74984932 6c1fcab85602 b9ade3d65856 2024-09-17T12:08:40.978 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (100s) 9s ago 15m 279M 2704M 19.3.0-4937-g74984932 6c1fcab85602 12ba3cdf09dd 2024-09-17T12:08:40.978 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (64s) 9s ago 14m 304M 2704M 19.3.0-4937-g74984932 6c1fcab85602 b0f172112f9e 2024-09-17T12:08:40.978 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (8m) 29s ago 18m 62.5M - 2.51.0 1d3b7f56885b 61b064ad6609 2024-09-17T12:08:41.453 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:41 smithi045 ceph-mon[63880]: from='client.34508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:08:41.453 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:08:41.454 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T12:08:41.454 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:08:41.454 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:08:41.454 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T12:08:41.454 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:08:41.454 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:08:41.454 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T12:08:41.454 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 8 2024-09-17T12:08:41.454 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:08:41.454 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T12:08:41.454 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:08:41.454 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:08:41.454 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T12:08:41.455 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 14 2024-09-17T12:08:41.455 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T12:08:41.455 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:08:41.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:41 smithi165 ceph-mon[49373]: from='client.34508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:08:41.827 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:08:41.828 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T12:08:41.828 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T12:08:41.828 INFO:teuthology.orchestra.run.smithi045.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T12:08:41.828 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [ 2024-09-17T12:08:41.828 INFO:teuthology.orchestra.run.smithi045.stdout: "crash", 2024-09-17T12:08:41.828 INFO:teuthology.orchestra.run.smithi045.stdout: "osd", 2024-09-17T12:08:41.828 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr", 2024-09-17T12:08:41.828 INFO:teuthology.orchestra.run.smithi045.stdout: "mds", 2024-09-17T12:08:41.828 INFO:teuthology.orchestra.run.smithi045.stdout: "mon" 2024-09-17T12:08:41.828 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-17T12:08:41.828 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "16/23 daemons upgraded", 2024-09-17T12:08:41.828 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "Currently upgrading nfs daemons", 2024-09-17T12:08:41.828 INFO:teuthology.orchestra.run.smithi045.stdout: "is_paused": false 2024-09-17T12:08:41.829 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:08:42.272 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_OK 2024-09-17T12:08:42.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:42 smithi165 ceph-mon[49373]: from='client.34512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:08:42.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:42 smithi165 ceph-mon[49373]: from='client.34516 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:08:42.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:42 smithi165 ceph-mon[49373]: pgmap v316: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s 2024-09-17T12:08:42.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:42 smithi165 ceph-mon[49373]: Evicting (and blocklisting) client session 14548 (172.21.15.45:0/1025805077) 2024-09-17T12:08:42.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:42 smithi165 ceph-mon[49373]: from='mds.? [v2:172.21.15.45:6834/2587321174,v1:172.21.15.45:6835/2587321174]' entity='mds.foofs.smithi045.olwaas' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.45:0/1025805077"} : dispatch 2024-09-17T12:08:42.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:42 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/3818675687' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:42.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:42 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:42.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:42 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:42.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:42 smithi045 ceph-mon[63880]: from='client.34512 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:08:42.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:42 smithi045 ceph-mon[63880]: from='client.34516 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:08:42.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:42 smithi045 ceph-mon[63880]: pgmap v316: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 170 B/s wr, 0 op/s 2024-09-17T12:08:42.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:42 smithi045 ceph-mon[63880]: Evicting (and blocklisting) client session 14548 (172.21.15.45:0/1025805077) 2024-09-17T12:08:42.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:42 smithi045 ceph-mon[63880]: from='mds.? [v2:172.21.15.45:6834/2587321174,v1:172.21.15.45:6835/2587321174]' entity='mds.foofs.smithi045.olwaas' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.45:0/1025805077"} : dispatch 2024-09-17T12:08:42.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:42 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/3818675687' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:42.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:42 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:42.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:42 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:43.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:43 smithi165 ceph-mon[49373]: from='client.34536 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:08:43.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:43 smithi165 ceph-mon[49373]: from='mds.? [v2:172.21.15.45:6834/2587321174,v1:172.21.15.45:6835/2587321174]' entity='mds.foofs.smithi045.olwaas' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.45:0/1025805077"}]': finished 2024-09-17T12:08:43.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:43 smithi165 ceph-mon[49373]: osdmap e151: 8 total, 8 up, 8 in 2024-09-17T12:08:43.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:43 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/4224799698' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:08:43.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:43 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:43.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:43 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi165.cvmryy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-17T12:08:43.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:43 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-17T12:08:43.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:43 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T12:08:43.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:43 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:08:43.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:43 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-17T12:08:43.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:43 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T12:08:43.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:43 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi165.cvmryy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-17T12:08:43.518 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:43 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:08:43.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:43 smithi045 ceph-mon[63880]: from='client.34536 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:08:43.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:43 smithi045 ceph-mon[63880]: from='mds.? [v2:172.21.15.45:6834/2587321174,v1:172.21.15.45:6835/2587321174]' entity='mds.foofs.smithi045.olwaas' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.45:0/1025805077"}]': finished 2024-09-17T12:08:43.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:43 smithi045 ceph-mon[63880]: osdmap e151: 8 total, 8 up, 8 in 2024-09-17T12:08:43.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:43 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/4224799698' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:08:43.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:43 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:43.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:43 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi165.cvmryy", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-17T12:08:43.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:43 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-17T12:08:43.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:43 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T12:08:43.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:43 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:08:43.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:43 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-17T12:08:43.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:43 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T12:08:43.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:43 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi165.cvmryy-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-17T12:08:43.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:43 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:08:44.467 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:44 smithi165 ceph-mon[49373]: Upgrade: Updating nfs.foo.1.0.smithi165.cvmryy (2/2) 2024-09-17T12:08:44.467 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:44 smithi165 ceph-mon[49373]: Creating key for client.nfs.foo.1.0.smithi165.cvmryy 2024-09-17T12:08:44.468 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:44 smithi165 ceph-mon[49373]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-17T12:08:44.468 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:44 smithi165 ceph-mon[49373]: Rados config object exists: conf-nfs.foo 2024-09-17T12:08:44.468 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:44 smithi165 ceph-mon[49373]: Creating key for client.nfs.foo.1.0.smithi165.cvmryy-rgw 2024-09-17T12:08:44.468 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:44 smithi165 ceph-mon[49373]: Bind address in nfs.foo.1.0.smithi165.cvmryy's ganesha conf is defaulting to empty 2024-09-17T12:08:44.468 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:44 smithi165 ceph-mon[49373]: Deploying daemon nfs.foo.1.0.smithi165.cvmryy on smithi165 2024-09-17T12:08:44.468 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:44 smithi165 ceph-mon[49373]: pgmap v318: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 614 B/s rd, 307 B/s wr, 0 op/s 2024-09-17T12:08:44.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:44 smithi045 ceph-mon[63880]: Upgrade: Updating nfs.foo.1.0.smithi165.cvmryy (2/2) 2024-09-17T12:08:44.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:44 smithi045 ceph-mon[63880]: Creating key for client.nfs.foo.1.0.smithi165.cvmryy 2024-09-17T12:08:44.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:44 smithi045 ceph-mon[63880]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-17T12:08:44.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:44 smithi045 ceph-mon[63880]: Rados config object exists: conf-nfs.foo 2024-09-17T12:08:44.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:44 smithi045 ceph-mon[63880]: Creating key for client.nfs.foo.1.0.smithi165.cvmryy-rgw 2024-09-17T12:08:44.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:44 smithi045 ceph-mon[63880]: Bind address in nfs.foo.1.0.smithi165.cvmryy's ganesha conf is defaulting to empty 2024-09-17T12:08:44.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:44 smithi045 ceph-mon[63880]: Deploying daemon nfs.foo.1.0.smithi165.cvmryy on smithi165 2024-09-17T12:08:44.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:44 smithi045 ceph-mon[63880]: pgmap v318: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 614 B/s rd, 307 B/s wr, 0 op/s 2024-09-17T12:08:46.280 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:46 smithi165 ceph-mon[49373]: pgmap v319: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-17T12:08:46.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:46 smithi045 ceph-mon[63880]: pgmap v319: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-17T12:08:47.474 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:47 smithi165 ceph-mon[49373]: Evicting (and blocklisting) client session 24369 (172.21.15.165:0/1603532166) 2024-09-17T12:08:47.474 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:47 smithi165 ceph-mon[49373]: from='mds.? [v2:172.21.15.45:6834/2587321174,v1:172.21.15.45:6835/2587321174]' entity='mds.foofs.smithi045.olwaas' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.165:0/1603532166"} : dispatch 2024-09-17T12:08:47.474 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:47 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:47.474 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:47 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:47.474 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:47 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:08:47.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:47 smithi045 ceph-mon[63880]: Evicting (and blocklisting) client session 24369 (172.21.15.165:0/1603532166) 2024-09-17T12:08:47.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:47 smithi045 ceph-mon[63880]: from='mds.? [v2:172.21.15.45:6834/2587321174,v1:172.21.15.45:6835/2587321174]' entity='mds.foofs.smithi045.olwaas' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.165:0/1603532166"} : dispatch 2024-09-17T12:08:47.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:47 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:47.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:47 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:47.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:47 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:08:48.472 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:48 smithi045 ceph-mon[63880]: pgmap v320: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-17T12:08:48.472 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:48 smithi045 ceph-mon[63880]: from='mds.? [v2:172.21.15.45:6834/2587321174,v1:172.21.15.45:6835/2587321174]' entity='mds.foofs.smithi045.olwaas' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.165:0/1603532166"}]': finished 2024-09-17T12:08:48.472 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:48 smithi045 ceph-mon[63880]: osdmap e152: 8 total, 8 up, 8 in 2024-09-17T12:08:48.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:48 smithi165 ceph-mon[49373]: pgmap v320: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-17T12:08:48.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:48 smithi165 ceph-mon[49373]: from='mds.? [v2:172.21.15.45:6834/2587321174,v1:172.21.15.45:6835/2587321174]' entity='mds.foofs.smithi045.olwaas' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.165:0/1603532166"}]': finished 2024-09-17T12:08:48.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:48 smithi165 ceph-mon[49373]: osdmap e152: 8 total, 8 up, 8 in 2024-09-17T12:08:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:50 smithi165 ceph-mon[49373]: pgmap v322: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.5 KiB/s wr, 8 op/s 2024-09-17T12:08:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:50 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:50 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:50.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:50 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:50.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:50 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:50.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:50 smithi045 ceph-mon[63880]: pgmap v322: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.5 KiB/s wr, 8 op/s 2024-09-17T12:08:50.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:50.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:50.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:50.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:50 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:52.394 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:52 smithi165 ceph-mon[49373]: pgmap v323: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.3 KiB/s rd, 2.2 KiB/s wr, 7 op/s 2024-09-17T12:08:52.394 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:52 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:52.395 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:52 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:52.395 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:52 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:08:52.395 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:52 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:52.395 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:52 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:52.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:52 smithi045 ceph-mon[63880]: pgmap v323: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.3 KiB/s rd, 2.2 KiB/s wr, 7 op/s 2024-09-17T12:08:52.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:52 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:52.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:52 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:52.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:52 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:08:52.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:52 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:52.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:52 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:53.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:53 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:53.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:53 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:53.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:53 smithi045 ceph-mon[63880]: pgmap v324: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.9 KiB/s wr, 6 op/s 2024-09-17T12:08:54.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:53 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:54.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:53 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:54.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:53 smithi165 ceph-mon[49373]: pgmap v324: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.9 KiB/s wr, 6 op/s 2024-09-17T12:08:56.301 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: pgmap v325: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s rd, 921 B/s wr, 2 op/s 2024-09-17T12:08:56.301 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:56.301 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:56.301 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:08:56.301 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:08:56.301 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:56.301 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:56.301 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:08:56.302 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.302 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.302 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.302 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.302 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.302 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.302 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.302 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.302 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.302 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.302 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.302 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:56.302 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi045.qochkw"} : dispatch 2024-09-17T12:08:56.302 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi045.qochkw"}]': finished 2024-09-17T12:08:56.303 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi165.cvmryy"} : dispatch 2024-09-17T12:08:56.303 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi165.cvmryy"}]': finished 2024-09-17T12:08:56.303 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.303 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:56.303 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.303 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.303 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:56 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.333 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: pgmap v325: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s rd, 921 B/s wr, 2 op/s 2024-09-17T12:08:56.333 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:56.333 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:56.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:08:56.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:08:56.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:56.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:56.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:08:56.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.334 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.335 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.335 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.335 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.335 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:56.335 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi045.qochkw"} : dispatch 2024-09-17T12:08:56.335 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi045.qochkw"}]': finished 2024-09-17T12:08:56.335 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi165.cvmryy"} : dispatch 2024-09-17T12:08:56.335 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi165.cvmryy"}]': finished 2024-09-17T12:08:56.335 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.335 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:08:56.335 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.335 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:56.335 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:56 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:08:57.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:57 smithi045 ceph-mon[63880]: Upgrade: Setting container_image for all nfs 2024-09-17T12:08:57.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:57 smithi045 ceph-mon[63880]: Upgrade: Setting container_image for all nvmeof 2024-09-17T12:08:57.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:57 smithi165 ceph-mon[49373]: Upgrade: Setting container_image for all nfs 2024-09-17T12:08:57.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:57 smithi165 ceph-mon[49373]: Upgrade: Setting container_image for all nvmeof 2024-09-17T12:08:58.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:58 smithi165 ceph-mon[49373]: pgmap v326: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s rd, 921 B/s wr, 2 op/s 2024-09-17T12:08:58.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:58 smithi045 ceph-mon[63880]: pgmap v326: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s rd, 921 B/s wr, 2 op/s 2024-09-17T12:08:59.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:59 smithi045 ceph-mon[63880]: Upgrade: Updating grafana.smithi045 2024-09-17T12:08:59.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:08:59 smithi045 ceph-mon[63880]: Deploying daemon grafana.smithi045 on smithi045 2024-09-17T12:08:59.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:59 smithi165 ceph-mon[49373]: Upgrade: Updating grafana.smithi045 2024-09-17T12:08:59.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:08:59 smithi165 ceph-mon[49373]: Deploying daemon grafana.smithi045 on smithi045 2024-09-17T12:09:00.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:00 smithi045 ceph-mon[63880]: pgmap v327: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s rd, 1.1 KiB/s wr, 3 op/s 2024-09-17T12:09:00.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:00 smithi165 ceph-mon[49373]: pgmap v327: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.0 KiB/s rd, 1.1 KiB/s wr, 3 op/s 2024-09-17T12:09:02.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:02 smithi045 ceph-mon[63880]: pgmap v328: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 341 B/s wr, 1 op/s 2024-09-17T12:09:02.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:02 smithi165 ceph-mon[49373]: pgmap v328: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 341 B/s wr, 1 op/s 2024-09-17T12:09:04.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:04 smithi045 ceph-mon[63880]: pgmap v329: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 341 B/s wr, 1 op/s 2024-09-17T12:09:04.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:04 smithi165 ceph-mon[49373]: pgmap v329: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 341 B/s wr, 1 op/s 2024-09-17T12:09:06.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:06 smithi165 ceph-mon[49373]: pgmap v330: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 341 B/s wr, 1 op/s 2024-09-17T12:09:06.531 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:06 smithi045 ceph-mon[63880]: pgmap v330: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 341 B/s wr, 1 op/s 2024-09-17T12:09:07.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:07 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:09:07.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:07 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:09:08.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:08 smithi045 ceph-mon[63880]: pgmap v331: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 341 B/s wr, 1 op/s 2024-09-17T12:09:08.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:08 smithi165 ceph-mon[49373]: pgmap v331: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 341 B/s wr, 1 op/s 2024-09-17T12:09:10.287 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:10 smithi045 ceph-mon[63880]: pgmap v332: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 937 B/s rd, 341 B/s wr, 1 op/s 2024-09-17T12:09:10.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:10 smithi165 ceph-mon[49373]: pgmap v332: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 937 B/s rd, 341 B/s wr, 1 op/s 2024-09-17T12:09:11.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:11 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:11.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:11 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:11.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:11 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:09:11.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:11 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:11.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:11 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:11.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:11 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:09:12.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:12 smithi165 ceph-mon[49373]: pgmap v333: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:12.578 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:12 smithi045 ceph-mon[63880]: pgmap v333: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:12.659 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T12:09:13.379 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T12:09:13.379 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (10m) 22s ago 19m 20.1M - 0.27.0 11f11916f8cd 2a2a54402d2a 2024-09-17T12:09:13.379 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (7m) 22s ago 19m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 e545cbf7736d 2024-09-17T12:09:13.379 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (7m) 24s ago 19m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 c1e89e114e4d 2024-09-17T12:09:13.379 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 starting - - - - 2024-09-17T12:09:13.379 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (68s) 22s ago 13m 27.9M - 19.3.0-4937-g74984932 6c1fcab85602 1db3f808b6e6 2024-09-17T12:09:13.379 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (45s) 24s ago 13m 13.2M - 19.3.0-4937-g74984932 6c1fcab85602 b13106022d43 2024-09-17T12:09:13.379 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:8443,9283,8765 running (9m) 22s ago 21m 526M - 19.3.0-4937-g74984932 6c1fcab85602 ae2f507bd67d 2024-09-17T12:09:13.379 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283,8765 running (8m) 24s ago 19m 458M - 19.3.0-4937-g74984932 6c1fcab85602 53403866652e 2024-09-17T12:09:13.379 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (8m) 22s ago 21m 63.3M 2048M 19.3.0-4937-g74984932 6c1fcab85602 737eec78e0f0 2024-09-17T12:09:13.379 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (8m) 24s ago 19m 49.4M 2048M 19.3.0-4937-g74984932 6c1fcab85602 249eedb4123e 2024-09-17T12:09:13.379 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (32s) 22s ago 13m 64.1M - 5.9 6c1fcab85602 6f9195b93fbf 2024-09-17T12:09:13.380 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (27s) 24s ago 13m 23.2M - 5.9 6c1fcab85602 7a0fd9788a6b 2024-09-17T12:09:13.380 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (10m) 22s ago 19m 11.7M - 1.7.0 72c9c2088986 989b5738cb69 2024-09-17T12:09:13.380 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (9m) 24s ago 18m 11.4M - 1.7.0 72c9c2088986 e1e8ffab3f21 2024-09-17T12:09:13.380 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (6m) 22s ago 18m 399M 1937M 19.3.0-4937-g74984932 6c1fcab85602 20dafd01bd12 2024-09-17T12:09:13.380 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (5m) 22s ago 17m 322M 1937M 19.3.0-4937-g74984932 6c1fcab85602 d39be5014418 2024-09-17T12:09:13.380 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (4m) 22s ago 17m 365M 1937M 19.3.0-4937-g74984932 6c1fcab85602 e13c2ecd369a 2024-09-17T12:09:13.380 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (4m) 22s ago 16m 381M 1937M 19.3.0-4937-g74984932 6c1fcab85602 ffd68312d242 2024-09-17T12:09:13.380 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (3m) 24s ago 16m 349M 2704M 19.3.0-4937-g74984932 6c1fcab85602 589de85559fd 2024-09-17T12:09:13.380 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (2m) 24s ago 16m 332M 2704M 19.3.0-4937-g74984932 6c1fcab85602 b9ade3d65856 2024-09-17T12:09:13.380 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (2m) 24s ago 15m 280M 2704M 19.3.0-4937-g74984932 6c1fcab85602 12ba3cdf09dd 2024-09-17T12:09:13.380 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (97s) 24s ago 15m 303M 2704M 19.3.0-4937-g74984932 6c1fcab85602 b0f172112f9e 2024-09-17T12:09:13.380 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (8m) 22s ago 18m 62.6M - 2.51.0 1d3b7f56885b 61b064ad6609 2024-09-17T12:09:13.816 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:09:13.817 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T12:09:13.817 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:09:13.817 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:09:13.817 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T12:09:13.817 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:09:13.817 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:09:13.818 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T12:09:13.818 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 8 2024-09-17T12:09:13.818 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:09:13.818 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T12:09:13.818 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:09:13.818 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:09:13.818 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T12:09:13.818 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 14 2024-09-17T12:09:13.819 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T12:09:13.819 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:09:14.181 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:09:14.181 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", 2024-09-17T12:09:14.181 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": true, 2024-09-17T12:09:14.181 INFO:teuthology.orchestra.run.smithi045.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T12:09:14.181 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [ 2024-09-17T12:09:14.181 INFO:teuthology.orchestra.run.smithi045.stdout: "crash", 2024-09-17T12:09:14.181 INFO:teuthology.orchestra.run.smithi045.stdout: "osd", 2024-09-17T12:09:14.181 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr", 2024-09-17T12:09:14.181 INFO:teuthology.orchestra.run.smithi045.stdout: "mds", 2024-09-17T12:09:14.181 INFO:teuthology.orchestra.run.smithi045.stdout: "mon", 2024-09-17T12:09:14.182 INFO:teuthology.orchestra.run.smithi045.stdout: "nfs" 2024-09-17T12:09:14.182 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-17T12:09:14.182 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": "18/23 daemons upgraded", 2024-09-17T12:09:14.182 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "Currently upgrading grafana daemons", 2024-09-17T12:09:14.182 INFO:teuthology.orchestra.run.smithi045.stdout: "is_paused": false 2024-09-17T12:09:14.182 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:09:14.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:14 smithi165 ceph-mon[49373]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:09:14.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:14 smithi165 ceph-mon[49373]: pgmap v334: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:14.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:14 smithi165 ceph-mon[49373]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:09:14.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:14 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/1503461195' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:14.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:14 smithi045 ceph-mon[63880]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:09:14.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:14 smithi045 ceph-mon[63880]: pgmap v334: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:14.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:14 smithi045 ceph-mon[63880]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:09:14.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:14 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/1503461195' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:14.632 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_OK 2024-09-17T12:09:15.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:15 smithi165 ceph-mon[49373]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:09:15.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:15 smithi165 ceph-mon[49373]: from='client.34580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:09:15.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:15 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/1213999429' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:09:15.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:15 smithi045 ceph-mon[63880]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:09:15.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:15 smithi045 ceph-mon[63880]: from='client.34580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:09:15.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:15 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/1213999429' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:09:16.415 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:16 smithi045 ceph-mon[63880]: pgmap v335: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:16.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:16 smithi165 ceph-mon[49373]: pgmap v335: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:18.013 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:17 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:18.013 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:17 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:18.013 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:17 smithi045 ceph-mon[63880]: pgmap v336: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:18.013 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:17 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:18.014 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:17 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:18.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:17 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:18.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:17 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:18.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:17 smithi165 ceph-mon[49373]: pgmap v336: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:18.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:17 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:18.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:17 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:20.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:20 smithi045 ceph-mon[63880]: pgmap v337: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:09:20.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:20 smithi165 ceph-mon[49373]: pgmap v337: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:09:22.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:22 smithi045 ceph-mon[63880]: pgmap v338: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:22.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:22 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:09:22.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:22 smithi165 ceph-mon[49373]: pgmap v338: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:22.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:22 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:09:24.245 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:24 smithi045 ceph-mon[63880]: pgmap v339: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:24.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:24 smithi165 ceph-mon[49373]: pgmap v339: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:25.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:25.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:25.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:09:25.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:09:25.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:25.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:25.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-17T12:09:25.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T12:09:25.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:09:25.591 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.591 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.591 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.591 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.591 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.591 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.591 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.591 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.591 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.591 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.591 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.591 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.591 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.592 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.592 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.592 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.592 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.592 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.592 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: Upgrade: Finalizing container_image settings 2024-09-17T12:09:25.592 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:25.592 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-17T12:09:25.592 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T12:09:25.593 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T12:09:25.593 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T12:09:25.593 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-17T12:09:25.593 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T12:09:25.593 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-17T12:09:25.593 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T12:09:25.593 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-17T12:09:25.593 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T12:09:25.593 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-17T12:09:25.593 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-17T12:09:25.593 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-17T12:09:25.593 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T12:09:25.593 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T12:09:25.594 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-17T12:09:25.594 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T12:09:25.594 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-17T12:09:25.594 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-17T12:09:25.594 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-17T12:09:25.594 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T12:09:25.594 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-17T12:09:25.594 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T12:09:25.594 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T12:09:25.594 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T12:09:25.595 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T12:09:25.595 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T12:09:25.595 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T12:09:25.595 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T12:09:25.595 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: Upgrade: Complete! 2024-09-17T12:09:25.595 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-17T12:09:25.595 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T12:09:25.595 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:09:25.595 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:09:25.595 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:09:25.595 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:25.595 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:25.595 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: pgmap v340: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:25.596 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:09:25.596 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:09:25.596 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:09:25.596 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:25 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:25.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:25.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:25.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:09:25.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:09:25.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:25.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:25.766 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-17T12:09:25.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T12:09:25.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:09:25.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.767 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:25.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: Upgrade: Finalizing container_image settings 2024-09-17T12:09:25.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:25.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-17T12:09:25.768 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T12:09:25.769 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T12:09:25.769 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T12:09:25.769 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-17T12:09:25.769 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T12:09:25.769 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-17T12:09:25.769 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T12:09:25.769 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-17T12:09:25.769 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T12:09:25.769 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-17T12:09:25.770 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-17T12:09:25.770 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-17T12:09:25.770 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T12:09:25.770 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T12:09:25.770 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-17T12:09:25.770 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T12:09:25.770 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-17T12:09:25.770 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-17T12:09:25.770 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-17T12:09:25.770 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T12:09:25.770 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-17T12:09:25.770 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T12:09:25.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T12:09:25.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T12:09:25.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T12:09:25.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T12:09:25.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T12:09:25.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-17T12:09:25.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: Upgrade: Complete! 2024-09-17T12:09:25.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-17T12:09:25.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T12:09:25.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:09:25.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:09:25.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:09:25.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:25.771 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:25.772 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: pgmap v340: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:25.772 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:09:25.772 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:09:25.772 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:09:25.772 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:25 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:26.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:26 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:26.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:26 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:27.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:27 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:27.839 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:27 smithi045 ceph-mon[63880]: pgmap v341: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:28.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:27 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:09:28.015 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:27 smithi165 ceph-mon[49373]: pgmap v341: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:29.592 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:29 smithi045 ceph-mon[63880]: pgmap v342: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:09:29.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:29 smithi165 ceph-mon[49373]: pgmap v342: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:09:32.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:32 smithi045 ceph-mon[63880]: pgmap v343: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:32.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:32 smithi165 ceph-mon[49373]: pgmap v343: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:34.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:34 smithi045 ceph-mon[63880]: pgmap v344: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:34.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:34 smithi165 ceph-mon[49373]: pgmap v344: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:36.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:36 smithi045 ceph-mon[63880]: pgmap v345: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:36.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:36 smithi165 ceph-mon[49373]: pgmap v345: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:37.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:37 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:09:37.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:37 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:09:38.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:38 smithi045 ceph-mon[63880]: pgmap v346: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:38.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:38 smithi165 ceph-mon[49373]: pgmap v346: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:40.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:40 smithi045 ceph-mon[63880]: pgmap v347: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:09:40.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:40 smithi165 ceph-mon[49373]: pgmap v347: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:09:42.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:42 smithi045 ceph-mon[63880]: pgmap v348: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:42.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:42 smithi165 ceph-mon[49373]: pgmap v348: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:44.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:44 smithi045 ceph-mon[63880]: pgmap v349: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:44.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:44 smithi165 ceph-mon[49373]: pgmap v349: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:45.799 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph orch ps' 2024-09-17T12:09:46.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:46 smithi045 ceph-mon[63880]: pgmap v350: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:46.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:46 smithi045 ceph-mon[63880]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:09:46.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:46 smithi165 ceph-mon[49373]: pgmap v350: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:46.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:46 smithi165 ceph-mon[49373]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:09:47.893 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T12:09:47.893 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (11m) 31s ago 20m 20.1M - 0.27.0 11f11916f8cd 2a2a54402d2a 2024-09-17T12:09:47.893 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (7m) 31s ago 20m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 e545cbf7736d 2024-09-17T12:09:47.893 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (7m) 58s ago 19m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 c1e89e114e4d 2024-09-17T12:09:47.894 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (38s) 31s ago 20m 34.5M - 10.4.0 c8b91775d855 bb1bcf3f3a84 2024-09-17T12:09:47.894 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (102s) 31s ago 14m 28.0M - 19.3.0-4937-g74984932 6c1fcab85602 1db3f808b6e6 2024-09-17T12:09:47.894 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (79s) 58s ago 14m 13.2M - 19.3.0-4937-g74984932 6c1fcab85602 b13106022d43 2024-09-17T12:09:47.894 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:8443,9283,8765 running (10m) 31s ago 21m 535M - 19.3.0-4937-g74984932 6c1fcab85602 ae2f507bd67d 2024-09-17T12:09:47.894 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283,8765 running (9m) 58s ago 19m 458M - 19.3.0-4937-g74984932 6c1fcab85602 53403866652e 2024-09-17T12:09:47.894 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (9m) 31s ago 21m 64.5M 2048M 19.3.0-4937-g74984932 6c1fcab85602 737eec78e0f0 2024-09-17T12:09:47.894 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (8m) 58s ago 19m 49.4M 2048M 19.3.0-4937-g74984932 6c1fcab85602 249eedb4123e 2024-09-17T12:09:47.894 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (67s) 31s ago 14m 65.0M - 5.9 6c1fcab85602 6f9195b93fbf 2024-09-17T12:09:47.894 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (62s) 58s ago 13m 23.2M - 5.9 6c1fcab85602 7a0fd9788a6b 2024-09-17T12:09:47.894 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (10m) 31s ago 20m 12.0M - 1.7.0 72c9c2088986 989b5738cb69 2024-09-17T12:09:47.894 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (10m) 58s ago 19m 11.4M - 1.7.0 72c9c2088986 e1e8ffab3f21 2024-09-17T12:09:47.894 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (7m) 31s ago 18m 400M 1937M 19.3.0-4937-g74984932 6c1fcab85602 20dafd01bd12 2024-09-17T12:09:47.894 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (6m) 31s ago 18m 322M 1937M 19.3.0-4937-g74984932 6c1fcab85602 d39be5014418 2024-09-17T12:09:47.894 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (5m) 31s ago 17m 365M 1937M 19.3.0-4937-g74984932 6c1fcab85602 e13c2ecd369a 2024-09-17T12:09:47.895 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (4m) 31s ago 17m 381M 1937M 19.3.0-4937-g74984932 6c1fcab85602 ffd68312d242 2024-09-17T12:09:47.895 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (4m) 58s ago 17m 349M 2704M 19.3.0-4937-g74984932 6c1fcab85602 589de85559fd 2024-09-17T12:09:47.895 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (3m) 58s ago 16m 332M 2704M 19.3.0-4937-g74984932 6c1fcab85602 b9ade3d65856 2024-09-17T12:09:47.895 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (2m) 58s ago 16m 280M 2704M 19.3.0-4937-g74984932 6c1fcab85602 12ba3cdf09dd 2024-09-17T12:09:47.895 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (2m) 58s ago 15m 303M 2704M 19.3.0-4937-g74984932 6c1fcab85602 b0f172112f9e 2024-09-17T12:09:47.895 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (9m) 31s ago 19m 62.6M - 2.51.0 1d3b7f56885b 61b064ad6609 2024-09-17T12:09:48.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:48 smithi045 ceph-mon[63880]: pgmap v351: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:48.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:48 smithi165 ceph-mon[49373]: pgmap v351: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:48.621 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph versions' 2024-09-17T12:09:49.197 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:49 smithi045 ceph-mon[63880]: from='client.34592 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:09:49.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:49 smithi165 ceph-mon[49373]: from='client.34592 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:09:50.296 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:50 smithi045 ceph-mon[63880]: pgmap v352: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:09:50.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:50 smithi165 ceph-mon[49373]: pgmap v352: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:09:50.858 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:09:50.858 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T12:09:50.858 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:09:50.858 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:09:50.858 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T12:09:50.859 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:09:50.859 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:09:50.859 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T12:09:50.859 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 8 2024-09-17T12:09:50.859 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:09:50.859 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T12:09:50.859 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:09:50.859 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:09:50.859 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T12:09:50.859 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 14 2024-09-17T12:09:50.859 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T12:09:50.860 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:09:51.233 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:51 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/1879773119' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:51.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:51 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/1879773119' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:09:51.583 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-17T12:09:52.212 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:52 smithi045 ceph-mon[63880]: pgmap v353: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:52.212 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:52 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:09:52.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:52 smithi165 ceph-mon[49373]: pgmap v353: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:52.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:52 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:09:53.461 INFO:teuthology.orchestra.run.smithi045.stdout:wait for servicemap items w/ changing names to refresh 2024-09-17T12:09:54.143 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'sleep 60' 2024-09-17T12:09:54.169 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:54 smithi045 ceph-mon[63880]: pgmap v354: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:54.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:54 smithi165 ceph-mon[49373]: pgmap v354: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:56.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:56 smithi165 ceph-mon[49373]: pgmap v355: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:56.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:56 smithi045 ceph-mon[63880]: pgmap v355: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:58.472 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:09:58 smithi045 ceph-mon[63880]: pgmap v356: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:09:58.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:09:58 smithi165 ceph-mon[49373]: pgmap v356: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:00.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:00 smithi165 ceph-mon[49373]: pgmap v357: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:10:00.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:00 smithi165 ceph-mon[49373]: overall HEALTH_OK 2024-09-17T12:10:00.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:00 smithi045 ceph-mon[63880]: pgmap v357: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:10:00.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:00 smithi045 ceph-mon[63880]: overall HEALTH_OK 2024-09-17T12:10:02.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:02 smithi165 ceph-mon[49373]: pgmap v358: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:02.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:02 smithi045 ceph-mon[63880]: pgmap v358: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:04.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:04 smithi165 ceph-mon[49373]: pgmap v359: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:04.592 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:04 smithi045 ceph-mon[63880]: pgmap v359: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:06.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:06 smithi165 ceph-mon[49373]: pgmap v360: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:06.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:06 smithi045 ceph-mon[63880]: pgmap v360: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:07.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:07 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:10:07.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:07 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:10:08.470 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:08 smithi045 ceph-mon[63880]: pgmap v361: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:08.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:08 smithi165 ceph-mon[49373]: pgmap v361: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:10.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:10 smithi165 ceph-mon[49373]: pgmap v362: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:10:10.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:10 smithi045 ceph-mon[63880]: pgmap v362: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:10:12.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:12 smithi165 ceph-mon[49373]: pgmap v363: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:12.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:12 smithi045 ceph-mon[63880]: pgmap v363: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:14.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:14 smithi165 ceph-mon[49373]: pgmap v364: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:14.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:14 smithi045 ceph-mon[63880]: pgmap v364: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:16.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:16 smithi165 ceph-mon[49373]: pgmap v365: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:16.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:16 smithi045 ceph-mon[63880]: pgmap v365: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:18.470 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:18 smithi045 ceph-mon[63880]: pgmap v366: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:18.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:18 smithi165 ceph-mon[49373]: pgmap v366: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:20.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:20 smithi165 ceph-mon[49373]: pgmap v367: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:10:20.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:20 smithi045 ceph-mon[63880]: pgmap v367: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:10:22.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:22 smithi165 ceph-mon[49373]: pgmap v368: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:22.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:22 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:10:22.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:22 smithi045 ceph-mon[63880]: pgmap v368: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:22.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:22 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:10:24.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:24 smithi165 ceph-mon[49373]: pgmap v369: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:24.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:24 smithi045 ceph-mon[63880]: pgmap v369: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:26.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:26 smithi165 ceph-mon[49373]: pgmap v370: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:26.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:26 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:10:26.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:26 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:10:26.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:26 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:10:26.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:26 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:10:26.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:26 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:10:26.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:26 smithi045 ceph-mon[63880]: pgmap v370: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:26.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:26 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:10:26.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:26 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:10:26.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:26 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:10:26.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:26 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:10:26.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:26 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:10:27.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:27 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:10:27.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:27 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:10:27.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:27 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:10:27.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:27 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:10:27.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:27 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:10:27.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:27 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:10:27.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:27 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:10:27.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:27 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:10:28.470 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:28 smithi045 ceph-mon[63880]: pgmap v371: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:28.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:28 smithi165 ceph-mon[49373]: pgmap v371: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:30.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:30 smithi165 ceph-mon[49373]: pgmap v372: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:10:30.590 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:30 smithi045 ceph-mon[63880]: pgmap v372: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:10:32.516 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:32 smithi165 ceph-mon[49373]: pgmap v373: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:32.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:32 smithi045 ceph-mon[63880]: pgmap v373: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:34.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:34 smithi165 ceph-mon[49373]: pgmap v374: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:34.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:34 smithi045 ceph-mon[63880]: pgmap v374: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:36.517 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:36 smithi165 ceph-mon[49373]: pgmap v375: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:36.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:36 smithi045 ceph-mon[63880]: pgmap v375: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:37.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:37 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:10:37.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:37 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:10:38.470 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:38 smithi045 ceph-mon[63880]: pgmap v376: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:38.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:38 smithi165 ceph-mon[49373]: pgmap v376: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:40.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:40 smithi165 ceph-mon[49373]: pgmap v377: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:10:40.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:40 smithi045 ceph-mon[63880]: pgmap v377: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:10:42.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:42 smithi165 ceph-mon[49373]: pgmap v378: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:42.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:42 smithi045 ceph-mon[63880]: pgmap v378: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:44.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:44 smithi165 ceph-mon[49373]: pgmap v379: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:44.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:44 smithi045 ceph-mon[63880]: pgmap v379: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:46.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:46 smithi165 ceph-mon[49373]: pgmap v380: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:46.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:46 smithi045 ceph-mon[63880]: pgmap v380: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:48.471 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:48 smithi045 ceph-mon[63880]: pgmap v381: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:48.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:48 smithi165 ceph-mon[49373]: pgmap v381: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:50.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:50 smithi165 ceph-mon[49373]: pgmap v382: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:10:50.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:50 smithi045 ceph-mon[63880]: pgmap v382: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:10:52.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:52 smithi165 ceph-mon[49373]: pgmap v383: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:52.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:52 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:10:52.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:52 smithi045 ceph-mon[63880]: pgmap v383: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:52.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:52 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:10:54.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:54 smithi165 ceph-mon[49373]: pgmap v384: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:54.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:54 smithi045 ceph-mon[63880]: pgmap v384: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:56.502 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:56 smithi045 ceph-mon[63880]: pgmap v385: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:56.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:56 smithi165 ceph-mon[49373]: pgmap v385: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:56.925 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph orch ps' 2024-09-17T12:10:58.471 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:10:58 smithi045 ceph-mon[63880]: pgmap v386: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:58.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:10:58 smithi165 ceph-mon[49373]: pgmap v386: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:10:59.195 INFO:teuthology.orchestra.run.smithi045.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T12:10:59.196 INFO:teuthology.orchestra.run.smithi045.stdout:alertmanager.smithi045 smithi045 *:9093,9094 running (12m) 102s ago 21m 20.1M - 0.27.0 11f11916f8cd 2a2a54402d2a 2024-09-17T12:10:59.196 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi045 smithi045 running (9m) 102s ago 21m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 e545cbf7736d 2024-09-17T12:10:59.196 INFO:teuthology.orchestra.run.smithi045.stdout:crash.smithi165 smithi165 running (8m) 2m ago 20m 6899k - 19.3.0-4937-g74984932 6c1fcab85602 c1e89e114e4d 2024-09-17T12:10:59.196 INFO:teuthology.orchestra.run.smithi045.stdout:grafana.smithi045 smithi045 *:3000 running (109s) 102s ago 21m 34.5M - 10.4.0 c8b91775d855 bb1bcf3f3a84 2024-09-17T12:10:59.196 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi045.olwaas smithi045 running (2m) 102s ago 15m 28.0M - 19.3.0-4937-g74984932 6c1fcab85602 1db3f808b6e6 2024-09-17T12:10:59.196 INFO:teuthology.orchestra.run.smithi045.stdout:mds.foofs.smithi165.czpmgd smithi165 running (2m) 2m ago 15m 13.2M - 19.3.0-4937-g74984932 6c1fcab85602 b13106022d43 2024-09-17T12:10:59.196 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi045.jtwyrp smithi045 *:8443,9283,8765 running (11m) 102s ago 22m 535M - 19.3.0-4937-g74984932 6c1fcab85602 ae2f507bd67d 2024-09-17T12:10:59.196 INFO:teuthology.orchestra.run.smithi045.stdout:mgr.smithi165.ykdszc smithi165 *:8443,9283,8765 running (10m) 2m ago 20m 458M - 19.3.0-4937-g74984932 6c1fcab85602 53403866652e 2024-09-17T12:10:59.196 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi045 smithi045 running (10m) 102s ago 22m 64.5M 2048M 19.3.0-4937-g74984932 6c1fcab85602 737eec78e0f0 2024-09-17T12:10:59.196 INFO:teuthology.orchestra.run.smithi045.stdout:mon.smithi165 smithi165 running (9m) 2m ago 20m 49.4M 2048M 19.3.0-4937-g74984932 6c1fcab85602 249eedb4123e 2024-09-17T12:10:59.196 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.0.0.smithi045.qochkw smithi045 *:2049 running (2m) 102s ago 15m 65.0M - 5.9 6c1fcab85602 6f9195b93fbf 2024-09-17T12:10:59.196 INFO:teuthology.orchestra.run.smithi045.stdout:nfs.foo.1.0.smithi165.cvmryy smithi165 *:2049 running (2m) 2m ago 15m 23.2M - 5.9 6c1fcab85602 7a0fd9788a6b 2024-09-17T12:10:59.196 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi045 smithi045 *:9100 running (12m) 102s ago 21m 12.0M - 1.7.0 72c9c2088986 989b5738cb69 2024-09-17T12:10:59.197 INFO:teuthology.orchestra.run.smithi045.stdout:node-exporter.smithi165 smithi165 *:9100 running (11m) 2m ago 20m 11.4M - 1.7.0 72c9c2088986 e1e8ffab3f21 2024-09-17T12:10:59.197 INFO:teuthology.orchestra.run.smithi045.stdout:osd.0 smithi045 running (8m) 102s ago 20m 400M 1937M 19.3.0-4937-g74984932 6c1fcab85602 20dafd01bd12 2024-09-17T12:10:59.197 INFO:teuthology.orchestra.run.smithi045.stdout:osd.1 smithi045 running (7m) 102s ago 19m 322M 1937M 19.3.0-4937-g74984932 6c1fcab85602 d39be5014418 2024-09-17T12:10:59.197 INFO:teuthology.orchestra.run.smithi045.stdout:osd.2 smithi045 running (6m) 102s ago 19m 365M 1937M 19.3.0-4937-g74984932 6c1fcab85602 e13c2ecd369a 2024-09-17T12:10:59.197 INFO:teuthology.orchestra.run.smithi045.stdout:osd.3 smithi045 running (5m) 102s ago 18m 381M 1937M 19.3.0-4937-g74984932 6c1fcab85602 ffd68312d242 2024-09-17T12:10:59.197 INFO:teuthology.orchestra.run.smithi045.stdout:osd.4 smithi165 running (5m) 2m ago 18m 349M 2704M 19.3.0-4937-g74984932 6c1fcab85602 589de85559fd 2024-09-17T12:10:59.197 INFO:teuthology.orchestra.run.smithi045.stdout:osd.5 smithi165 running (4m) 2m ago 17m 332M 2704M 19.3.0-4937-g74984932 6c1fcab85602 b9ade3d65856 2024-09-17T12:10:59.197 INFO:teuthology.orchestra.run.smithi045.stdout:osd.6 smithi165 running (3m) 2m ago 17m 280M 2704M 19.3.0-4937-g74984932 6c1fcab85602 12ba3cdf09dd 2024-09-17T12:10:59.197 INFO:teuthology.orchestra.run.smithi045.stdout:osd.7 smithi165 running (3m) 2m ago 17m 303M 2704M 19.3.0-4937-g74984932 6c1fcab85602 b0f172112f9e 2024-09-17T12:10:59.197 INFO:teuthology.orchestra.run.smithi045.stdout:prometheus.smithi045 smithi045 *:9095 running (10m) 102s ago 20m 62.6M - 2.51.0 1d3b7f56885b 61b064ad6609 2024-09-17T12:10:59.914 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph orch upgrade status' 2024-09-17T12:11:00.470 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:00 smithi045 ceph-mon[63880]: pgmap v387: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:11:00.470 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:00 smithi045 ceph-mon[63880]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:11:00.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:00 smithi165 ceph-mon[49373]: pgmap v387: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:11:00.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:00 smithi165 ceph-mon[49373]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:11:02.060 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:11:02.060 INFO:teuthology.orchestra.run.smithi045.stdout: "target_image": null, 2024-09-17T12:11:02.060 INFO:teuthology.orchestra.run.smithi045.stdout: "in_progress": false, 2024-09-17T12:11:02.060 INFO:teuthology.orchestra.run.smithi045.stdout: "which": "", 2024-09-17T12:11:02.061 INFO:teuthology.orchestra.run.smithi045.stdout: "services_complete": [], 2024-09-17T12:11:02.061 INFO:teuthology.orchestra.run.smithi045.stdout: "progress": null, 2024-09-17T12:11:02.061 INFO:teuthology.orchestra.run.smithi045.stdout: "message": "", 2024-09-17T12:11:02.061 INFO:teuthology.orchestra.run.smithi045.stdout: "is_paused": false 2024-09-17T12:11:02.061 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:11:02.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:02 smithi045 ceph-mon[63880]: pgmap v388: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:11:02.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:02 smithi165 ceph-mon[49373]: pgmap v388: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:11:02.792 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph health detail' 2024-09-17T12:11:03.385 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:03 smithi045 ceph-mon[63880]: from='client.34604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:11:03.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:03 smithi165 ceph-mon[49373]: from='client.34604 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:11:04.336 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:04 smithi045 ceph-mon[63880]: pgmap v389: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:11:04.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:04 smithi165 ceph-mon[49373]: pgmap v389: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:11:04.885 INFO:teuthology.orchestra.run.smithi045.stdout:HEALTH_OK 2024-09-17T12:11:05.497 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:05 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/1465888117' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:11:05.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:05 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/1465888117' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-17T12:11:05.521 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph versions' 2024-09-17T12:11:06.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:06 smithi165 ceph-mon[49373]: pgmap v390: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:11:06.572 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:06 smithi045 ceph-mon[63880]: pgmap v390: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:11:07.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:07 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:11:07.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:07 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:11:07.745 INFO:teuthology.orchestra.run.smithi045.stdout:{ 2024-09-17T12:11:07.746 INFO:teuthology.orchestra.run.smithi045.stdout: "mon": { 2024-09-17T12:11:07.746 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:11:07.746 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:11:07.746 INFO:teuthology.orchestra.run.smithi045.stdout: "mgr": { 2024-09-17T12:11:07.746 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:11:07.746 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:11:07.746 INFO:teuthology.orchestra.run.smithi045.stdout: "osd": { 2024-09-17T12:11:07.746 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 8 2024-09-17T12:11:07.746 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:11:07.746 INFO:teuthology.orchestra.run.smithi045.stdout: "mds": { 2024-09-17T12:11:07.746 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 2 2024-09-17T12:11:07.746 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:11:07.747 INFO:teuthology.orchestra.run.smithi045.stdout: "overall": { 2024-09-17T12:11:07.747 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)": 14 2024-09-17T12:11:07.747 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T12:11:07.747 INFO:teuthology.orchestra.run.smithi045.stdout:} 2024-09-17T12:11:08.424 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:08 smithi045 ceph-mon[63880]: pgmap v391: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:11:08.424 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:08 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/2270512141' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:11:08.447 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-17T12:11:08.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:08 smithi165 ceph-mon[49373]: pgmap v391: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:11:08.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:08 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/2270512141' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:11:10.340 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:10 smithi045 ceph-mon[63880]: pgmap v392: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:11:10.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:10 smithi165 ceph-mon[49373]: pgmap v392: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:11:10.640 INFO:teuthology.orchestra.run.smithi045.stdout:true 2024-09-17T12:11:11.337 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -e sha1=7498493279982dfe87f00616198a5967475f6169 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-17T12:11:11.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:11 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/2271857546' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:11:11.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:11 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/2271857546' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:11:12.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:12 smithi165 ceph-mon[49373]: pgmap v393: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:11:12.527 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:12 smithi045 ceph-mon[63880]: pgmap v393: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:11:13.390 INFO:teuthology.orchestra.run.smithi045.stdout: "ceph version 19.3.0-4937-g74984932 (7498493279982dfe87f00616198a5967475f6169) squid (dev)" 2024-09-17T12:11:14.107 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-09-17T12:11:14.108 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-09-17T12:11:14.108 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-17T12:11:14.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:14 smithi045 ceph-mon[63880]: pgmap v394: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:11:14.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:14 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/1907544666' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:11:14.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:14 smithi165 ceph-mon[49373]: pgmap v394: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:11:14.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:14 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/1907544666' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-17T12:11:16.042 INFO:teuthology.orchestra.run.smithi045.stdout: 2024-09-17T12:11:16.042 INFO:teuthology.orchestra.run.smithi045.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-17T11:48:51.619752Z", "last_refresh": "2024-09-17T12:09:16.749475Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-17T11:48:45.016215Z", "last_refresh": "2024-09-17T12:08:49.075387Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-09-17T11:48:47.912727Z", "last_refresh": "2024-09-17T12:09:16.749613Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-17T11:55:22.373606Z", "last_refresh": "2024-09-17T12:08:49.076326Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-17T11:48:43.055030Z", "last_refresh": "2024-09-17T12:08:49.075527Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi045:172.21.15.45=smithi045", "smithi165:172.21.15.165=smithi165"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-17T11:49:34.840638Z", "last_refresh": "2024-09-17T12:08:49.075648Z", "running": 2, "size": 2}}, {"events": ["2024-09-17T12:10:26.375574Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-09-17T11:55:42.772251Z", "last_refresh": "2024-09-17T12:08:49.076439Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-17T11:48:50.361555Z", "last_refresh": "2024-09-17T12:08:49.075757Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb", "last_refresh": "2024-09-17T12:08:49.075866Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-17T11:48:46.212310Z", "last_refresh": "2024-09-17T12:09:16.749749Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-17T12:11:16.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:16 smithi045 ceph-mon[63880]: pgmap v395: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:11:16.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:16 smithi165 ceph-mon[49373]: pgmap v395: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:11:16.734 INFO:tasks.cephadm:nfs.foo has 2/2 2024-09-17T12:11:16.735 DEBUG:teuthology.parallel:result is None 2024-09-17T12:11:16.735 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-17T12:11:16.746 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi045.front.sepia.ceph.com 2024-09-17T12:11:16.746 DEBUG:teuthology.orchestra.run.smithi045:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-09-17T12:11:16.774 INFO:teuthology.orchestra.run.smithi045.stderr:+ umount /mnt/foo 2024-09-17T12:11:16.799 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T12:11:16.809 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi045.front.sepia.ceph.com 2024-09-17T12:11:16.810 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-09-17T12:11:17.404 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:17 smithi045 ceph-mon[63880]: from='client.34624 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T12:11:17.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:17 smithi165 ceph-mon[49373]: from='client.34624 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T12:11:18.392 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:18 smithi045 ceph-mon[63880]: pgmap v396: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:11:18.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:18 smithi165 ceph-mon[49373]: pgmap v396: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:11:18.918 INFO:teuthology.orchestra.run.smithi045.stdout: "foo" 2024-09-17T12:11:19.592 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export ls foo --detailed' 2024-09-17T12:11:20.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:20 smithi165 ceph-mon[49373]: from='client.34628 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:11:20.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:20 smithi165 ceph-mon[49373]: pgmap v397: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:11:20.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:20 smithi045 ceph-mon[63880]: from='client.34628 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:11:20.589 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:20 smithi045 ceph-mon[63880]: pgmap v397: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T12:11:21.738 INFO:teuthology.orchestra.run.smithi045.stdout:[ 2024-09-17T12:11:21.738 INFO:teuthology.orchestra.run.smithi045.stdout: { 2024-09-17T12:11:21.738 INFO:teuthology.orchestra.run.smithi045.stdout: "access_type": "RW", 2024-09-17T12:11:21.738 INFO:teuthology.orchestra.run.smithi045.stdout: "clients": [], 2024-09-17T12:11:21.739 INFO:teuthology.orchestra.run.smithi045.stdout: "cluster_id": "foo", 2024-09-17T12:11:21.739 INFO:teuthology.orchestra.run.smithi045.stdout: "export_id": 1, 2024-09-17T12:11:21.739 INFO:teuthology.orchestra.run.smithi045.stdout: "fsal": { 2024-09-17T12:11:21.739 INFO:teuthology.orchestra.run.smithi045.stdout: "fs_name": "foofs", 2024-09-17T12:11:21.739 INFO:teuthology.orchestra.run.smithi045.stdout: "name": "CEPH", 2024-09-17T12:11:21.739 INFO:teuthology.orchestra.run.smithi045.stdout: "user_id": "nfs.foo.1" 2024-09-17T12:11:21.739 INFO:teuthology.orchestra.run.smithi045.stdout: }, 2024-09-17T12:11:21.739 INFO:teuthology.orchestra.run.smithi045.stdout: "path": "/", 2024-09-17T12:11:21.739 INFO:teuthology.orchestra.run.smithi045.stdout: "protocols": [ 2024-09-17T12:11:21.739 INFO:teuthology.orchestra.run.smithi045.stdout: 4 2024-09-17T12:11:21.739 INFO:teuthology.orchestra.run.smithi045.stdout: ], 2024-09-17T12:11:21.739 INFO:teuthology.orchestra.run.smithi045.stdout: "pseudo": "/fake", 2024-09-17T12:11:21.740 INFO:teuthology.orchestra.run.smithi045.stdout: "security_label": true, 2024-09-17T12:11:21.740 INFO:teuthology.orchestra.run.smithi045.stdout: "squash": "none", 2024-09-17T12:11:21.740 INFO:teuthology.orchestra.run.smithi045.stdout: "transports": [ 2024-09-17T12:11:21.740 INFO:teuthology.orchestra.run.smithi045.stdout: "TCP" 2024-09-17T12:11:21.740 INFO:teuthology.orchestra.run.smithi045.stdout: ] 2024-09-17T12:11:21.740 INFO:teuthology.orchestra.run.smithi045.stdout: } 2024-09-17T12:11:21.740 INFO:teuthology.orchestra.run.smithi045.stdout:] 2024-09-17T12:11:22.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:22 smithi045 ceph-mon[63880]: pgmap v398: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:11:22.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:22 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:11:22.409 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- bash -c 'rados -p .nfs --all ls -' 2024-09-17T12:11:22.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:22 smithi165 ceph-mon[49373]: pgmap v398: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T12:11:22.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:22 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-17T12:11:23.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:23 smithi045 ceph-mon[63880]: from='client.34632 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:11:23.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:23 smithi165 ceph-mon[49373]: from='client.34632 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T12:11:24.201 INFO:teuthology.orchestra.run.smithi045.stdout:foo rec-0000000000000004:nfs.foo.1 2024-09-17T12:11:24.201 INFO:teuthology.orchestra.run.smithi045.stdout:foo grace 2024-09-17T12:11:24.201 INFO:teuthology.orchestra.run.smithi045.stdout:foo rec-0000000000000004:nfs.foo.0 2024-09-17T12:11:24.201 INFO:teuthology.orchestra.run.smithi045.stdout:foo conf-nfs.foo 2024-09-17T12:11:24.201 INFO:teuthology.orchestra.run.smithi045.stdout:foo export-1 2024-09-17T12:11:24.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:24 smithi165 ceph-mon[49373]: pgmap v399: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T12:11:24.515 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:24 smithi165 ceph-mon[49373]: mgrmap e44: smithi045.jtwyrp(active, since 10m), standbys: smithi165.ykdszc 2024-09-17T12:11:24.530 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:24 smithi045 ceph-mon[63880]: pgmap v399: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T12:11:24.530 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:24 smithi045 ceph-mon[63880]: mgrmap e44: smithi045.jtwyrp(active, since 10m), standbys: smithi165.ykdszc 2024-09-17T12:11:24.836 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T12:11:24.849 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi045.front.sepia.ceph.com 2024-09-17T12:11:24.849 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a12537de-74ea-11ef-bceb-c7b262605968 -- bash -c 'set -ex 2024-09-17T12:11:24.849 DEBUG:teuthology.orchestra.run.smithi045:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-09-17T12:11:24.849 DEBUG:teuthology.orchestra.run.smithi045:> ' 2024-09-17T12:11:25.508 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:25 smithi045 ceph-mon[63880]: pgmap v400: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T12:11:25.765 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:25 smithi165 ceph-mon[49373]: pgmap v400: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T12:11:26.707 INFO:teuthology.orchestra.run.smithi045.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-09-17T12:11:26.965 INFO:teuthology.orchestra.run.smithi045.stderr:+ [[ 8 -gt 2 ]] 2024-09-17T12:11:27.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:26 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:11:27.339 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:26 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-17T12:11:27.871 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-17T12:11:27.881 INFO:tasks.cephadm:Teardown begin 2024-09-17T12:11:27.881 DEBUG:teuthology.orchestra.run.smithi045:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T12:11:27.908 DEBUG:teuthology.orchestra.run.smithi165:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T12:11:27.940 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-17T12:11:27.941 DEBUG:teuthology.orchestra.run.smithi045:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-17T12:11:27.964 DEBUG:teuthology.orchestra.run.smithi165:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-17T12:11:27.998 INFO:tasks.cephadm:Stopping all daemons... 2024-09-17T12:11:27.998 INFO:tasks.cephadm.mon.smithi045:Stopping mon.smithi045... 2024-09-17T12:11:27.998 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ceph-a12537de-74ea-11ef-bceb-c7b262605968@mon.smithi045 2024-09-17T12:11:28.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:27 smithi165 ceph-mon[49373]: from='client.? 172.21.15.45:0/2329705500' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-17T12:11:28.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:27 smithi165 ceph-mon[49373]: pgmap v401: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T12:11:28.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:27 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:11:28.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:27 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:11:28.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:27 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:11:28.266 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:27 smithi165 ceph-mon[49373]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:11:28.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:27 smithi045 ceph-mon[63880]: from='client.? 172.21.15.45:0/2329705500' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-17T12:11:28.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:27 smithi045 ceph-mon[63880]: pgmap v401: 97 pgs: 97 active+clean; 224 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T12:11:28.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:27 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-17T12:11:28.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:27 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-17T12:11:28.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:27 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:11:28.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:27 smithi045 ceph-mon[63880]: from='mgr.34102 172.21.15.45:0/4068326029' entity='mgr.smithi045.jtwyrp' 2024-09-17T12:11:28.311 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:28 smithi045 systemd[1]: Stopping Ceph mon.smithi045 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:11:28.577 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:28 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045[63876]: 2024-09-17T12:11:28.308+0000 7f87eb206640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi045 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T12:11:28.577 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:28 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045[63876]: 2024-09-17T12:11:28.308+0000 7f87eb206640 -1 mon.smithi045@0(leader) e3 *** Got Signal Terminated *** 2024-09-17T12:11:28.577 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:28 smithi045 podman[93519]: 2024-09-17 12:11:28.421449516 +0000 UTC m=+0.218815971 container died 737eec78e0f09fb20b773507fa81cd666ce003d854ab3b58021f4490c035c057 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=main-7498493, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-17T12:11:28.840 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:28 smithi045 podman[93519]: 2024-09-17 12:11:28.800271025 +0000 UTC m=+0.597637480 container cleanup 737eec78e0f09fb20b773507fa81cd666ce003d854ab3b58021f4490c035c057 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-17T12:11:28.841 INFO:journalctl@ceph.mon.smithi045.smithi045.stdout:Sep 17 12:11:28 smithi045 bash[93519]: ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi045 2024-09-17T12:11:29.193 DEBUG:teuthology.orchestra.run.smithi045:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a12537de-74ea-11ef-bceb-c7b262605968@mon.smithi045.service' 2024-09-17T12:11:29.245 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T12:11:29.248 INFO:tasks.cephadm.mon.smithi045:Stopped mon.smithi045 2024-09-17T12:11:29.249 INFO:tasks.cephadm.mon.smithi165:Stopping mon.smithi165... 2024-09-17T12:11:29.249 DEBUG:teuthology.orchestra.run.smithi165:> sudo systemctl stop ceph-a12537de-74ea-11ef-bceb-c7b262605968@mon.smithi165 2024-09-17T12:11:29.643 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:29 smithi165 systemd[1]: Stopping Ceph mon.smithi165 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:11:29.967 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:29 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi165[49369]: 2024-09-17T12:11:29.639+0000 7fbb8fc96640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi165 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T12:11:29.967 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:29 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi165[49369]: 2024-09-17T12:11:29.639+0000 7fbb8fc96640 -1 mon.smithi165@1(peon) e3 *** Got Signal Terminated *** 2024-09-17T12:11:29.967 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:29 smithi165 podman[68320]: 2024-09-17 12:11:29.69235042 +0000 UTC m=+0.249911193 container died 249eedb4123e0e62c0f100b2b23178f4290b25e31d098f9bbafd3adbd13a71e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi165, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-17T12:11:30.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:30 smithi165 podman[68320]: 2024-09-17 12:11:30.140515408 +0000 UTC m=+0.698076179 container cleanup 249eedb4123e0e62c0f100b2b23178f4290b25e31d098f9bbafd3adbd13a71e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi165, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T12:11:30.265 INFO:journalctl@ceph.mon.smithi165.smithi165.stdout:Sep 17 12:11:30 smithi165 bash[68320]: ceph-a12537de-74ea-11ef-bceb-c7b262605968-mon-smithi165 2024-09-17T12:11:30.622 DEBUG:teuthology.orchestra.run.smithi165:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a12537de-74ea-11ef-bceb-c7b262605968@mon.smithi165.service' 2024-09-17T12:11:30.663 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T12:11:30.663 INFO:tasks.cephadm.mon.smithi165:Stopped mon.smithi165 2024-09-17T12:11:30.663 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-17T12:11:30.663 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.0 2024-09-17T12:11:30.981 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:11:30 smithi045 systemd[1]: Stopping Ceph osd.0 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:11:31.339 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:11:30 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0[68810]: 2024-09-17T12:11:30.979+0000 7f2f4fcdc640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T12:11:31.339 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:11:30 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0[68810]: 2024-09-17T12:11:30.979+0000 7f2f4fcdc640 -1 osd.0 152 *** Got signal Terminated *** 2024-09-17T12:11:31.339 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:11:30 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0[68810]: 2024-09-17T12:11:30.979+0000 7f2f4fcdc640 -1 osd.0 152 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T12:11:36.657 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:11:36 smithi045 podman[93651]: 2024-09-17 12:11:36.37136168 +0000 UTC m=+5.507157942 container died 20dafd01bd124690d2f84a35f7d611f5ed67c6a13a79002077b2475e976ead9d (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-17T12:11:37.023 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:11:36 smithi045 podman[93651]: 2024-09-17 12:11:36.762713899 +0000 UTC m=+5.898510156 container cleanup 20dafd01bd124690d2f84a35f7d611f5ed67c6a13a79002077b2475e976ead9d (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T12:11:37.023 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:11:36 smithi045 bash[93651]: ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0 2024-09-17T12:11:37.339 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:11:37 smithi045 podman[93667]: 2024-09-17 12:11:37.022592079 +0000 UTC m=+0.651103268 container remove 20dafd01bd124690d2f84a35f7d611f5ed67c6a13a79002077b2475e976ead9d (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0, RELEASE=main-7498493, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T12:11:37.668 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:11:37 smithi045 podman[93755]: 2024-09-17 12:11:37.301498383 +0000 UTC m=+0.017233013 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:11:37.668 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:11:37 smithi045 podman[93755]: 2024-09-17 12:11:37.423563621 +0000 UTC m=+0.139298241 container create 3decdea3aad2eefc68bdc1594cbf54358ea73255f942f98909ecfbeeb62c6ecc (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, RELEASE=main-7498493, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-17T12:11:37.668 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:11:37 smithi045 podman[93755]: 2024-09-17 12:11:37.667731775 +0000 UTC m=+0.383466408 container init 3decdea3aad2eefc68bdc1594cbf54358ea73255f942f98909ecfbeeb62c6ecc (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=main-7498493, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-17T12:11:38.089 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:11:37 smithi045 podman[93755]: 2024-09-17 12:11:37.671384495 +0000 UTC m=+0.387119119 container start 3decdea3aad2eefc68bdc1594cbf54358ea73255f942f98909ecfbeeb62c6ecc (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, RELEASE=main-7498493) 2024-09-17T12:11:38.090 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:11:37 smithi045 podman[93755]: 2024-09-17 12:11:37.767734945 +0000 UTC m=+0.483469576 container attach 3decdea3aad2eefc68bdc1594cbf54358ea73255f942f98909ecfbeeb62c6ecc (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-17T12:11:38.090 INFO:journalctl@ceph.osd.0.smithi045.stdout:Sep 17 12:11:37 smithi045 podman[93755]: 2024-09-17 12:11:37.874637946 +0000 UTC m=+0.590372575 container died 3decdea3aad2eefc68bdc1594cbf54358ea73255f942f98909ecfbeeb62c6ecc (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=main-7498493, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-17T12:11:38.496 DEBUG:teuthology.orchestra.run.smithi045:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.0.service' 2024-09-17T12:11:38.543 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T12:11:38.544 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-17T12:11:38.544 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-17T12:11:38.544 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.1 2024-09-17T12:11:38.839 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:11:38 smithi045 systemd[1]: Stopping Ceph osd.1 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:11:39.339 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:11:38 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1[71152]: 2024-09-17T12:11:38.890+0000 7f76c639b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T12:11:39.339 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:11:38 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1[71152]: 2024-09-17T12:11:38.890+0000 7f76c639b640 -1 osd.1 152 *** Got signal Terminated *** 2024-09-17T12:11:39.339 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:11:38 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1[71152]: 2024-09-17T12:11:38.890+0000 7f76c639b640 -1 osd.1 152 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T12:11:44.643 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:11:44 smithi045 podman[93857]: 2024-09-17 12:11:44.382254257 +0000 UTC m=+5.604359534 container died d39be5014418be8f139753fafa73dc2e93264a9d233b4eae1427d15abc47bd8f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-17T12:11:44.959 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:11:44 smithi045 podman[93857]: 2024-09-17 12:11:44.716054151 +0000 UTC m=+5.938159423 container cleanup d39be5014418be8f139753fafa73dc2e93264a9d233b4eae1427d15abc47bd8f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-7498493, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-17T12:11:44.960 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:11:44 smithi045 bash[93857]: ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1 2024-09-17T12:11:45.339 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:11:44 smithi045 podman[93890]: 2024-09-17 12:11:44.958807953 +0000 UTC m=+0.577894018 container remove d39be5014418be8f139753fafa73dc2e93264a9d233b4eae1427d15abc47bd8f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=main-7498493, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-17T12:11:45.696 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:11:45 smithi045 podman[93977]: 2024-09-17 12:11:45.311936169 +0000 UTC m=+0.017138130 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:11:45.697 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:11:45 smithi045 podman[93977]: 2024-09-17 12:11:45.442948287 +0000 UTC m=+0.148150274 container create 9b943e17284674389e710e376ef80b31e548674b73213393701dc1dab74dab53 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-17T12:11:45.697 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:11:45 smithi045 podman[93977]: 2024-09-17 12:11:45.695716118 +0000 UTC m=+0.400918072 container init 9b943e17284674389e710e376ef80b31e548674b73213393701dc1dab74dab53 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, RELEASE=main-7498493, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T12:11:46.090 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:11:45 smithi045 podman[93977]: 2024-09-17 12:11:45.699446914 +0000 UTC m=+0.404648859 container start 9b943e17284674389e710e376ef80b31e548674b73213393701dc1dab74dab53 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=main-7498493, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-17T12:11:46.090 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:11:45 smithi045 podman[93977]: 2024-09-17 12:11:45.778974268 +0000 UTC m=+0.484176213 container attach 9b943e17284674389e710e376ef80b31e548674b73213393701dc1dab74dab53 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T12:11:46.090 INFO:journalctl@ceph.osd.1.smithi045.stdout:Sep 17 12:11:45 smithi045 podman[93977]: 2024-09-17 12:11:45.899886117 +0000 UTC m=+0.605088064 container died 9b943e17284674389e710e376ef80b31e548674b73213393701dc1dab74dab53 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, RELEASE=main-7498493, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-17T12:11:46.516 DEBUG:teuthology.orchestra.run.smithi045:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.1.service' 2024-09-17T12:11:46.558 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T12:11:46.558 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-17T12:11:46.558 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-17T12:11:46.558 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.2 2024-09-17T12:11:46.839 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:11:46 smithi045 systemd[1]: Stopping Ceph osd.2 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:11:47.339 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:11:46 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2[73583]: 2024-09-17T12:11:46.876+0000 7ff29adaf640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T12:11:47.340 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:11:46 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2[73583]: 2024-09-17T12:11:46.876+0000 7ff29adaf640 -1 osd.2 152 *** Got signal Terminated *** 2024-09-17T12:11:47.340 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:11:46 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2[73583]: 2024-09-17T12:11:46.876+0000 7ff29adaf640 -1 osd.2 152 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T12:11:52.780 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:11:52 smithi045 podman[94080]: 2024-09-17 12:11:52.481492599 +0000 UTC m=+5.697658530 container died e13c2ecd369ac53b2f9288427f2a071d5c802d3effe0cbc79e58bf6d2319857c (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731) 2024-09-17T12:11:53.089 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:11:52 smithi045 podman[94080]: 2024-09-17 12:11:52.859856102 +0000 UTC m=+6.076022022 container cleanup e13c2ecd369ac53b2f9288427f2a071d5c802d3effe0cbc79e58bf6d2319857c (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T12:11:53.089 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:11:52 smithi045 bash[94080]: ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2 2024-09-17T12:11:53.496 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:11:53 smithi045 podman[94097]: 2024-09-17 12:11:53.144894756 +0000 UTC m=+0.656945676 container remove e13c2ecd369ac53b2f9288427f2a071d5c802d3effe0cbc79e58bf6d2319857c (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, RELEASE=main-7498493, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T12:11:53.816 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:11:53 smithi045 podman[94184]: 2024-09-17 12:11:53.398940465 +0000 UTC m=+0.016630395 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:11:53.816 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:11:53 smithi045 podman[94184]: 2024-09-17 12:11:53.512465902 +0000 UTC m=+0.130155831 container create f5f8fb3e427613562b082d6eb71e4ebaa66b1dcdf01cfbfec9b5d33c43dd4e4f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=main-7498493, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T12:11:53.816 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:11:53 smithi045 podman[94184]: 2024-09-17 12:11:53.731575435 +0000 UTC m=+0.349265369 container init f5f8fb3e427613562b082d6eb71e4ebaa66b1dcdf01cfbfec9b5d33c43dd4e4f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=main-7498493, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-17T12:11:53.816 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:11:53 smithi045 podman[94184]: 2024-09-17 12:11:53.73489105 +0000 UTC m=+0.352580980 container start f5f8fb3e427613562b082d6eb71e4ebaa66b1dcdf01cfbfec9b5d33c43dd4e4f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=main-7498493, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T12:11:53.816 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:11:53 smithi045 podman[94184]: 2024-09-17 12:11:53.814951491 +0000 UTC m=+0.432641421 container attach f5f8fb3e427613562b082d6eb71e4ebaa66b1dcdf01cfbfec9b5d33c43dd4e4f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-deactivate, RELEASE=main-7498493, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-17T12:11:54.089 INFO:journalctl@ceph.osd.2.smithi045.stdout:Sep 17 12:11:53 smithi045 podman[94184]: 2024-09-17 12:11:53.927048674 +0000 UTC m=+0.544738613 container died f5f8fb3e427613562b082d6eb71e4ebaa66b1dcdf01cfbfec9b5d33c43dd4e4f (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, org.label-schema.vendor=CentOS) 2024-09-17T12:11:54.518 DEBUG:teuthology.orchestra.run.smithi045:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.2.service' 2024-09-17T12:11:54.559 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T12:11:54.559 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-17T12:11:54.559 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-17T12:11:54.560 DEBUG:teuthology.orchestra.run.smithi045:> sudo systemctl stop ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.3 2024-09-17T12:11:54.839 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:11:54 smithi045 systemd[1]: Stopping Ceph osd.3 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:11:55.339 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:11:54 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3[75938]: 2024-09-17T12:11:54.938+0000 7fafcda37640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T12:11:55.339 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:11:54 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3[75938]: 2024-09-17T12:11:54.938+0000 7fafcda37640 -1 osd.3 152 *** Got signal Terminated *** 2024-09-17T12:11:55.339 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:11:54 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3[75938]: 2024-09-17T12:11:54.938+0000 7fafcda37640 -1 osd.3 152 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T12:11:58.339 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:11:58 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3[75938]: 2024-09-17T12:11:58.053+0000 7fafc6044640 -1 osd.3 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:34.049082+0000 front 2024-09-17T12:11:34.048904+0000 (oldest deadline 2024-09-17T12:11:57.549024+0000) 2024-09-17T12:11:58.696 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:11:58 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:11:58.410+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:11:58.697 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:11:58 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:11:58.313+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:11:58.949 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:11:58 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4[53245]: 2024-09-17T12:11:58.840+0000 7f4e95617640 -1 osd.4 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:33.963511+0000 front 2024-09-17T12:11:33.963646+0000 (oldest deadline 2024-09-17T12:11:58.063555+0000) 2024-09-17T12:11:59.589 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:11:59 smithi045 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3[75938]: 2024-09-17T12:11:59.090+0000 7fafc6044640 -1 osd.3 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:34.049082+0000 front 2024-09-17T12:11:34.048904+0000 (oldest deadline 2024-09-17T12:11:57.549024+0000) 2024-09-17T12:11:59.765 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:11:59 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:11:59.373+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:11:59.765 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:11:59 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:11:59.315+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:00.265 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:11:59 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4[53245]: 2024-09-17T12:11:59.808+0000 7f4e95617640 -1 osd.4 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:33.963511+0000 front 2024-09-17T12:11:33.963646+0000 (oldest deadline 2024-09-17T12:11:58.063555+0000) 2024-09-17T12:12:00.667 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:12:00 smithi045 podman[94286]: 2024-09-17 12:12:00.388524803 +0000 UTC m=+5.590818932 container died ffd68312d2425a78fa50437bb2a043ea84ce606a0fe2480228dfa70d1708603a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, RELEASE=main-7498493, maintainer=Guillaume Abrioux ) 2024-09-17T12:12:00.765 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:00 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:00.295+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:00.765 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:00 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:00.379+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:01.024 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:12:00 smithi045 podman[94286]: 2024-09-17 12:12:00.746721708 +0000 UTC m=+5.949015826 container cleanup ffd68312d2425a78fa50437bb2a043ea84ce606a0fe2480228dfa70d1708603a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-17T12:12:01.024 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:12:00 smithi045 bash[94286]: ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3 2024-09-17T12:12:01.265 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:12:00 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4[53245]: 2024-09-17T12:12:00.773+0000 7f4e95617640 -1 osd.4 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:33.963511+0000 front 2024-09-17T12:11:33.963646+0000 (oldest deadline 2024-09-17T12:11:58.063555+0000) 2024-09-17T12:12:01.339 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:12:01 smithi045 podman[94302]: 2024-09-17 12:12:01.02289704 +0000 UTC m=+0.636793672 container remove ffd68312d2425a78fa50437bb2a043ea84ce606a0fe2480228dfa70d1708603a (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T12:12:01.635 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:12:01 smithi045 podman[94392]: 2024-09-17 12:12:01.27751988 +0000 UTC m=+0.016614658 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:12:01.635 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:12:01 smithi045 podman[94392]: 2024-09-17 12:12:01.381902403 +0000 UTC m=+0.120997163 container create 571df12e2996528734b061b32a1521cba8355d22a0eabd796e396de9f0d76551 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493) 2024-09-17T12:12:01.635 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:12:01 smithi045 podman[94392]: 2024-09-17 12:12:01.634370872 +0000 UTC m=+0.373465642 container init 571df12e2996528734b061b32a1521cba8355d22a0eabd796e396de9f0d76551 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=main-7498493, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T12:12:01.763 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:01 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:01.441+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:31.334870+0000 front 2024-09-17T12:11:31.334910+0000 (oldest deadline 2024-09-17T12:12:01.335232+0000) 2024-09-17T12:12:01.763 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:01 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:01.354+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:01.763 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:01 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:01.345+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:02.016 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:12:01 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4[53245]: 2024-09-17T12:12:01.761+0000 7f4e95617640 -1 osd.4 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:33.963511+0000 front 2024-09-17T12:11:33.963646+0000 (oldest deadline 2024-09-17T12:11:58.063555+0000) 2024-09-17T12:12:02.089 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:12:01 smithi045 podman[94392]: 2024-09-17 12:12:01.639016104 +0000 UTC m=+0.378110865 container start 571df12e2996528734b061b32a1521cba8355d22a0eabd796e396de9f0d76551 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-17T12:12:02.089 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:12:01 smithi045 podman[94392]: 2024-09-17 12:12:01.726014381 +0000 UTC m=+0.465109156 container attach 571df12e2996528734b061b32a1521cba8355d22a0eabd796e396de9f0d76551 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-deactivate, RELEASE=main-7498493, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-17T12:12:02.089 INFO:journalctl@ceph.osd.3.smithi045.stdout:Sep 17 12:12:01 smithi045 podman[94392]: 2024-09-17 12:12:01.829638238 +0000 UTC m=+0.568732999 container died 571df12e2996528734b061b32a1521cba8355d22a0eabd796e396de9f0d76551 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-3-deactivate, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-17T12:12:02.515 DEBUG:teuthology.orchestra.run.smithi045:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.3.service' 2024-09-17T12:12:02.558 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T12:12:02.559 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-17T12:12:02.559 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-17T12:12:02.559 DEBUG:teuthology.orchestra.run.smithi165:> sudo systemctl stop ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.4 2024-09-17T12:12:02.588 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:02 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:02.432+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:31.334870+0000 front 2024-09-17T12:11:31.334910+0000 (oldest deadline 2024-09-17T12:12:01.335232+0000) 2024-09-17T12:12:02.589 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:02 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:02.327+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:02.589 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:02 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:02.366+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:02.851 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:12:02 smithi165 systemd[1]: Stopping Ceph osd.4 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:12:02.851 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:12:02 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4[53245]: 2024-09-17T12:12:02.734+0000 7f4e95617640 -1 osd.4 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:33.963511+0000 front 2024-09-17T12:11:33.963646+0000 (oldest deadline 2024-09-17T12:11:58.063555+0000) 2024-09-17T12:12:02.851 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:12:02 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4[53245]: 2024-09-17T12:12:02.849+0000 7f4e99002640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T12:12:03.265 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:12:02 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4[53245]: 2024-09-17T12:12:02.849+0000 7f4e99002640 -1 osd.4 152 *** Got signal Terminated *** 2024-09-17T12:12:03.265 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:12:02 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4[53245]: 2024-09-17T12:12:02.849+0000 7f4e99002640 -1 osd.4 152 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T12:12:03.765 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:03 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:03.301+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:03.765 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:03 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:03.418+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:31.334870+0000 front 2024-09-17T12:11:31.334910+0000 (oldest deadline 2024-09-17T12:12:01.335232+0000) 2024-09-17T12:12:03.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:03 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:03.350+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:04.268 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:12:03 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4[53245]: 2024-09-17T12:12:03.766+0000 7f4e95617640 -1 osd.4 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:33.963511+0000 front 2024-09-17T12:11:33.963646+0000 (oldest deadline 2024-09-17T12:11:58.063555+0000) 2024-09-17T12:12:04.765 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:04 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:04.392+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:31.334870+0000 front 2024-09-17T12:11:31.334910+0000 (oldest deadline 2024-09-17T12:12:01.335232+0000) 2024-09-17T12:12:04.765 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:04 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:04.368+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:04.766 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:04 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:04.335+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:05.265 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:12:04 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4[53245]: 2024-09-17T12:12:04.814+0000 7f4e95617640 -1 osd.4 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:33.963511+0000 front 2024-09-17T12:11:33.963646+0000 (oldest deadline 2024-09-17T12:11:58.063555+0000) 2024-09-17T12:12:05.765 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:05 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:05.383+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:31.334870+0000 front 2024-09-17T12:11:31.334910+0000 (oldest deadline 2024-09-17T12:12:01.335232+0000) 2024-09-17T12:12:05.766 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:05 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:05.296+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:05.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:05 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:05.389+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:06.265 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:12:05 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4[53245]: 2024-09-17T12:12:05.852+0000 7f4e95617640 -1 osd.4 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:33.963511+0000 front 2024-09-17T12:11:33.963646+0000 (oldest deadline 2024-09-17T12:11:58.063555+0000) 2024-09-17T12:12:06.765 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:06 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:06.364+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:31.334870+0000 front 2024-09-17T12:11:31.334910+0000 (oldest deadline 2024-09-17T12:12:01.335232+0000) 2024-09-17T12:12:06.765 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:06 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:06.364+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.335759+0000 front 2024-09-17T12:11:41.335686+0000 (oldest deadline 2024-09-17T12:12:06.035599+0000) 2024-09-17T12:12:06.766 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:06 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:06.269+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:06.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:06 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:06.368+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:06.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:06 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:06.368+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:07.265 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:12:06 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4[53245]: 2024-09-17T12:12:06.866+0000 7f4e95617640 -1 osd.4 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:33.963511+0000 front 2024-09-17T12:11:33.963646+0000 (oldest deadline 2024-09-17T12:11:58.063555+0000) 2024-09-17T12:12:07.765 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:07 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:07.311+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:07.766 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:07 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:07.395+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:31.334870+0000 front 2024-09-17T12:11:31.334910+0000 (oldest deadline 2024-09-17T12:12:01.335232+0000) 2024-09-17T12:12:07.766 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:07 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:07.395+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.335759+0000 front 2024-09-17T12:11:41.335686+0000 (oldest deadline 2024-09-17T12:12:06.035599+0000) 2024-09-17T12:12:07.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:07 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:07.347+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:07.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:07 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:07.348+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:08.694 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:12:08 smithi165 podman[68468]: 2024-09-17 12:12:08.41166024 +0000 UTC m=+5.656251715 container died 589de85559fd665c9f3190da54822cafc42714b0a558935878318242edfab795 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=main-7498493, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T12:12:08.694 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:08 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:08.407+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:31.334870+0000 front 2024-09-17T12:11:31.334910+0000 (oldest deadline 2024-09-17T12:12:01.335232+0000) 2024-09-17T12:12:08.694 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:08 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:08.407+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.335759+0000 front 2024-09-17T12:11:41.335686+0000 (oldest deadline 2024-09-17T12:12:06.035599+0000) 2024-09-17T12:12:08.695 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:08 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:08.343+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:08.695 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:08 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:08.343+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.464253+0000 front 2024-09-17T12:11:41.464076+0000 (oldest deadline 2024-09-17T12:12:07.864800+0000) 2024-09-17T12:12:08.695 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:08 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:08.396+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:08.695 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:08 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:08.396+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:09.015 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:12:08 smithi165 podman[68468]: 2024-09-17 12:12:08.858134413 +0000 UTC m=+6.102725888 container cleanup 589de85559fd665c9f3190da54822cafc42714b0a558935878318242edfab795 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=main-7498493, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T12:12:09.015 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:12:08 smithi165 bash[68468]: ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4 2024-09-17T12:12:09.396 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:09 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:09.298+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:09.396 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:09 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:09.298+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.464253+0000 front 2024-09-17T12:11:41.464076+0000 (oldest deadline 2024-09-17T12:12:07.864800+0000) 2024-09-17T12:12:09.396 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:12:09 smithi165 podman[68498]: 2024-09-17 12:12:09.143127714 +0000 UTC m=+0.729630577 container remove 589de85559fd665c9f3190da54822cafc42714b0a558935878318242edfab795 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4, RELEASE=main-7498493, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-17T12:12:09.396 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:09 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:09.389+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:09.397 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:09 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:09.389+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:09.730 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:09 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:09.394+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:31.334870+0000 front 2024-09-17T12:11:31.334910+0000 (oldest deadline 2024-09-17T12:12:01.335232+0000) 2024-09-17T12:12:09.731 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:09 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:09.394+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.335759+0000 front 2024-09-17T12:11:41.335686+0000 (oldest deadline 2024-09-17T12:12:06.035599+0000) 2024-09-17T12:12:09.731 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:12:09 smithi165 podman[68588]: 2024-09-17 12:12:09.380296411 +0000 UTC m=+0.018914178 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:12:09.731 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:12:09 smithi165 podman[68588]: 2024-09-17 12:12:09.510964113 +0000 UTC m=+0.149581920 container create 1d58e0d0a4f67d5e48da995d737ff40112b8f1efc6a8574f9ab42197072a12e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, RELEASE=main-7498493, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-17T12:12:09.731 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:12:09 smithi165 podman[68588]: 2024-09-17 12:12:09.729990847 +0000 UTC m=+0.368608607 container init 1d58e0d0a4f67d5e48da995d737ff40112b8f1efc6a8574f9ab42197072a12e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T12:12:10.016 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:12:09 smithi165 podman[68588]: 2024-09-17 12:12:09.733583134 +0000 UTC m=+0.372200893 container start 1d58e0d0a4f67d5e48da995d737ff40112b8f1efc6a8574f9ab42197072a12e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-17T12:12:10.016 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:12:09 smithi165 podman[68588]: 2024-09-17 12:12:09.804671926 +0000 UTC m=+0.443289697 container attach 1d58e0d0a4f67d5e48da995d737ff40112b8f1efc6a8574f9ab42197072a12e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T12:12:10.016 INFO:journalctl@ceph.osd.4.smithi165.stdout:Sep 17 12:12:09 smithi165 podman[68588]: 2024-09-17 12:12:09.940916498 +0000 UTC m=+0.579534264 container died 1d58e0d0a4f67d5e48da995d737ff40112b8f1efc6a8574f9ab42197072a12e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=main-7498493) 2024-09-17T12:12:10.515 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:10 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:10.273+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:10.515 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:10 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:10.273+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.464253+0000 front 2024-09-17T12:11:41.464076+0000 (oldest deadline 2024-09-17T12:12:07.864800+0000) 2024-09-17T12:12:10.516 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:10 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:10.443+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:31.334870+0000 front 2024-09-17T12:11:31.334910+0000 (oldest deadline 2024-09-17T12:12:01.335232+0000) 2024-09-17T12:12:10.516 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:10 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:10.443+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.335759+0000 front 2024-09-17T12:11:41.335686+0000 (oldest deadline 2024-09-17T12:12:06.035599+0000) 2024-09-17T12:12:10.516 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:10 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:10.418+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:10.516 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:10 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:10.418+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:10.543 DEBUG:teuthology.orchestra.run.smithi165:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.4.service' 2024-09-17T12:12:10.626 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T12:12:10.628 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-17T12:12:10.628 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-17T12:12:10.628 DEBUG:teuthology.orchestra.run.smithi165:> sudo systemctl stop ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.5 2024-09-17T12:12:10.780 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:10 smithi165 systemd[1]: Stopping Ceph osd.5 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:12:11.242 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:10 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:10.977+0000 7f1bc7d80640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T12:12:11.242 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:10 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:10.977+0000 7f1bc7d80640 -1 osd.5 152 *** Got signal Terminated *** 2024-09-17T12:12:11.242 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:10 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:10.977+0000 7f1bc7d80640 -1 osd.5 152 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T12:12:11.515 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:11 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:11.369+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:11.515 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:11 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:11.369+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:11.515 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:11 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:11.488+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:31.334870+0000 front 2024-09-17T12:11:31.334910+0000 (oldest deadline 2024-09-17T12:12:01.335232+0000) 2024-09-17T12:12:11.516 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:11 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:11.488+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.335759+0000 front 2024-09-17T12:11:41.335686+0000 (oldest deadline 2024-09-17T12:12:06.035599+0000) 2024-09-17T12:12:11.516 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:11 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:11.240+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:11.516 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:11 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:11.240+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.464253+0000 front 2024-09-17T12:11:41.464076+0000 (oldest deadline 2024-09-17T12:12:07.864800+0000) 2024-09-17T12:12:12.515 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:12 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:12.238+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:12.516 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:12 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:12.238+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.464253+0000 front 2024-09-17T12:11:41.464076+0000 (oldest deadline 2024-09-17T12:12:07.864800+0000) 2024-09-17T12:12:12.516 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:12 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:12.340+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:12.516 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:12 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:12.340+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:12.516 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:12 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:12.470+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:31.334870+0000 front 2024-09-17T12:11:31.334910+0000 (oldest deadline 2024-09-17T12:12:01.335232+0000) 2024-09-17T12:12:12.516 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:12 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:12.470+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.335759+0000 front 2024-09-17T12:11:41.335686+0000 (oldest deadline 2024-09-17T12:12:06.035599+0000) 2024-09-17T12:12:13.515 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:13 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:13.358+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:13.515 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:13 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:13.358+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:13.516 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:13 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:13.429+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:31.334870+0000 front 2024-09-17T12:11:31.334910+0000 (oldest deadline 2024-09-17T12:12:01.335232+0000) 2024-09-17T12:12:13.516 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:13 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:13.429+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.335759+0000 front 2024-09-17T12:11:41.335686+0000 (oldest deadline 2024-09-17T12:12:06.035599+0000) 2024-09-17T12:12:13.516 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:13 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:13.247+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:13.516 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:13 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:13.247+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.464253+0000 front 2024-09-17T12:11:41.464076+0000 (oldest deadline 2024-09-17T12:12:07.864800+0000) 2024-09-17T12:12:14.515 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:14 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:14.239+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:14.515 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:14 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:14.239+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.464253+0000 front 2024-09-17T12:11:41.464076+0000 (oldest deadline 2024-09-17T12:12:07.864800+0000) 2024-09-17T12:12:14.516 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:14 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:14.328+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:14.516 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:14 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:14.328+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:14.516 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:14 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:14.476+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:31.334870+0000 front 2024-09-17T12:11:31.334910+0000 (oldest deadline 2024-09-17T12:12:01.335232+0000) 2024-09-17T12:12:14.516 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:14 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:14.476+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.335759+0000 front 2024-09-17T12:11:41.335686+0000 (oldest deadline 2024-09-17T12:12:06.035599+0000) 2024-09-17T12:12:15.503 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:15 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:15.285+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:15.504 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:15 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:15.285+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:15.504 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:15 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:15.238+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:15.504 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:15 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:15.238+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.464253+0000 front 2024-09-17T12:11:41.464076+0000 (oldest deadline 2024-09-17T12:12:07.864800+0000) 2024-09-17T12:12:15.765 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:15 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:15.502+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:31.334870+0000 front 2024-09-17T12:11:31.334910+0000 (oldest deadline 2024-09-17T12:12:01.335232+0000) 2024-09-17T12:12:15.765 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:15 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5[55251]: 2024-09-17T12:12:15.502+0000 7f1bc038d640 -1 osd.5 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.335759+0000 front 2024-09-17T12:11:41.335686+0000 (oldest deadline 2024-09-17T12:12:06.035599+0000) 2024-09-17T12:12:16.516 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:16 smithi165 podman[68692]: 2024-09-17 12:12:16.413905095 +0000 UTC m=+5.556488596 container died b9ade3d65856493d1283258db5ad7a33d7bc28668473f732d75595469a5cecba (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-7498493, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-17T12:12:16.516 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:16 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:16.236+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:16.517 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:16 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:16.236+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.464253+0000 front 2024-09-17T12:11:41.464076+0000 (oldest deadline 2024-09-17T12:12:07.864800+0000) 2024-09-17T12:12:16.517 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:16 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:16.236+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6822 osd.2 since back 2024-09-17T12:11:50.165997+0000 front 2024-09-17T12:11:50.166012+0000 (oldest deadline 2024-09-17T12:12:15.465678+0000) 2024-09-17T12:12:16.517 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:16 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:16.322+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:16.517 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:16 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:16.322+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:17.015 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:16 smithi165 podman[68692]: 2024-09-17 12:12:16.792614639 +0000 UTC m=+5.935198136 container cleanup b9ade3d65856493d1283258db5ad7a33d7bc28668473f732d75595469a5cecba (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T12:12:17.015 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:16 smithi165 bash[68692]: ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5 2024-09-17T12:12:17.515 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:17 smithi165 podman[68721]: 2024-09-17 12:12:17.070471272 +0000 UTC m=+0.652222441 container remove b9ade3d65856493d1283258db5ad7a33d7bc28668473f732d75595469a5cecba (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=main-7498493, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-17T12:12:17.516 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:17 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:17.275+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:17.516 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:17 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:17.275+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.464253+0000 front 2024-09-17T12:11:41.464076+0000 (oldest deadline 2024-09-17T12:12:07.864800+0000) 2024-09-17T12:12:17.516 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:17 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:17.275+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6822 osd.2 since back 2024-09-17T12:11:50.165997+0000 front 2024-09-17T12:11:50.166012+0000 (oldest deadline 2024-09-17T12:12:15.465678+0000) 2024-09-17T12:12:17.516 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:17 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:17.315+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:17.516 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:17 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:17.315+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:17.516 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:17 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:17.315+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6822 osd.2 since back 2024-09-17T12:11:51.493690+0000 front 2024-09-17T12:11:51.493670+0000 (oldest deadline 2024-09-17T12:12:16.793582+0000) 2024-09-17T12:12:17.808 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:17 smithi165 podman[68806]: 2024-09-17 12:12:17.452336654 +0000 UTC m=+0.015882582 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:12:17.809 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:17 smithi165 podman[68806]: 2024-09-17 12:12:17.562812424 +0000 UTC m=+0.126358340 container create ce5dd65a1114c34c29f52eef0e1187ca5cd03bd388c6b06bce18a13feee50661 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=main-7498493, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-17T12:12:17.809 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:17 smithi165 podman[68806]: 2024-09-17 12:12:17.807830724 +0000 UTC m=+0.371376651 container init ce5dd65a1114c34c29f52eef0e1187ca5cd03bd388c6b06bce18a13feee50661 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-deactivate, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-17T12:12:18.243 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:17 smithi165 podman[68806]: 2024-09-17 12:12:17.811278201 +0000 UTC m=+0.374824124 container start ce5dd65a1114c34c29f52eef0e1187ca5cd03bd388c6b06bce18a13feee50661 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-deactivate, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, RELEASE=main-7498493, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-17T12:12:18.243 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:17 smithi165 podman[68806]: 2024-09-17 12:12:17.899476221 +0000 UTC m=+0.463022159 container attach ce5dd65a1114c34c29f52eef0e1187ca5cd03bd388c6b06bce18a13feee50661 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=main-7498493, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T12:12:18.243 INFO:journalctl@ceph.osd.5.smithi165.stdout:Sep 17 12:12:17 smithi165 podman[68806]: 2024-09-17 12:12:17.994926682 +0000 UTC m=+0.558472600 container died ce5dd65a1114c34c29f52eef0e1187ca5cd03bd388c6b06bce18a13feee50661 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-7498493, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-17T12:12:18.515 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:18 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:18.242+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:18.515 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:18 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:18.242+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.464253+0000 front 2024-09-17T12:11:41.464076+0000 (oldest deadline 2024-09-17T12:12:07.864800+0000) 2024-09-17T12:12:18.515 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:18 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:18.242+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6822 osd.2 since back 2024-09-17T12:11:50.165997+0000 front 2024-09-17T12:11:50.166012+0000 (oldest deadline 2024-09-17T12:12:15.465678+0000) 2024-09-17T12:12:18.516 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:18 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:18.314+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:18.516 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:18 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:18.315+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:18.516 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:18 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:18.315+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6822 osd.2 since back 2024-09-17T12:11:51.493690+0000 front 2024-09-17T12:11:51.493670+0000 (oldest deadline 2024-09-17T12:12:16.793582+0000) 2024-09-17T12:12:18.597 DEBUG:teuthology.orchestra.run.smithi165:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.5.service' 2024-09-17T12:12:18.636 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T12:12:18.636 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-17T12:12:18.636 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-17T12:12:18.637 DEBUG:teuthology.orchestra.run.smithi165:> sudo systemctl stop ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.6 2024-09-17T12:12:18.965 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:18 smithi165 systemd[1]: Stopping Ceph osd.6 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:12:19.265 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:18 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:18.963+0000 7f967150e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T12:12:19.265 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:18 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:18.963+0000 7f967150e640 -1 osd.6 152 *** Got signal Terminated *** 2024-09-17T12:12:19.265 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:18 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:18.963+0000 7f967150e640 -1 osd.6 152 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T12:12:19.765 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:19 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:19.287+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:19.765 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:19 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:19.287+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.464253+0000 front 2024-09-17T12:11:41.464076+0000 (oldest deadline 2024-09-17T12:12:07.864800+0000) 2024-09-17T12:12:19.765 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:19 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:19.287+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6822 osd.2 since back 2024-09-17T12:11:50.165997+0000 front 2024-09-17T12:11:50.166012+0000 (oldest deadline 2024-09-17T12:12:15.465678+0000) 2024-09-17T12:12:19.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:19 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:19.357+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:19.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:19 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:19.357+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:19.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:19 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:19.357+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6822 osd.2 since back 2024-09-17T12:11:51.493690+0000 front 2024-09-17T12:11:51.493670+0000 (oldest deadline 2024-09-17T12:12:16.793582+0000) 2024-09-17T12:12:20.765 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:20 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:20.298+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:20.765 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:20 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:20.298+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.464253+0000 front 2024-09-17T12:11:41.464076+0000 (oldest deadline 2024-09-17T12:12:07.864800+0000) 2024-09-17T12:12:20.765 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:20 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:20.298+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6822 osd.2 since back 2024-09-17T12:11:50.165997+0000 front 2024-09-17T12:11:50.166012+0000 (oldest deadline 2024-09-17T12:12:15.465678+0000) 2024-09-17T12:12:20.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:20 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:20.350+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:20.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:20 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:20.350+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:20.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:20 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:20.350+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6822 osd.2 since back 2024-09-17T12:11:51.493690+0000 front 2024-09-17T12:11:51.493670+0000 (oldest deadline 2024-09-17T12:12:16.793582+0000) 2024-09-17T12:12:21.515 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:21 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:21.304+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:21.515 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:21 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:21.304+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.464253+0000 front 2024-09-17T12:11:41.464076+0000 (oldest deadline 2024-09-17T12:12:07.864800+0000) 2024-09-17T12:12:21.516 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:21 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:21.304+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6822 osd.2 since back 2024-09-17T12:11:50.165997+0000 front 2024-09-17T12:11:50.166012+0000 (oldest deadline 2024-09-17T12:12:15.465678+0000) 2024-09-17T12:12:21.516 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:21 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:21.361+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:21.516 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:21 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:21.361+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:21.516 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:21 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:21.361+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6822 osd.2 since back 2024-09-17T12:11:51.493690+0000 front 2024-09-17T12:11:51.493670+0000 (oldest deadline 2024-09-17T12:12:16.793582+0000) 2024-09-17T12:12:22.765 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:22 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:22.305+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:22.765 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:22 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:22.305+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.464253+0000 front 2024-09-17T12:11:41.464076+0000 (oldest deadline 2024-09-17T12:12:07.864800+0000) 2024-09-17T12:12:22.765 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:22 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:22.305+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6822 osd.2 since back 2024-09-17T12:11:50.165997+0000 front 2024-09-17T12:11:50.166012+0000 (oldest deadline 2024-09-17T12:12:15.465678+0000) 2024-09-17T12:12:22.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:22 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:22.357+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:22.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:22 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:22.357+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:22.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:22 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:22.357+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6822 osd.2 since back 2024-09-17T12:11:51.493690+0000 front 2024-09-17T12:11:51.493670+0000 (oldest deadline 2024-09-17T12:12:16.793582+0000) 2024-09-17T12:12:23.765 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:23 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:23.319+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.763034+0000 front 2024-09-17T12:11:32.763159+0000 (oldest deadline 2024-09-17T12:11:58.063067+0000) 2024-09-17T12:12:23.765 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:23 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:23.319+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:41.464253+0000 front 2024-09-17T12:11:41.464076+0000 (oldest deadline 2024-09-17T12:12:07.864800+0000) 2024-09-17T12:12:23.765 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:23 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6[57247]: 2024-09-17T12:12:23.319+0000 7f966db23640 -1 osd.6 152 heartbeat_check: no reply from 172.21.15.45:6822 osd.2 since back 2024-09-17T12:11:50.165997+0000 front 2024-09-17T12:11:50.166012+0000 (oldest deadline 2024-09-17T12:12:15.465678+0000) 2024-09-17T12:12:23.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:23 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:23.334+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:23.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:23 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:23.334+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:23.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:23 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:23.334+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6822 osd.2 since back 2024-09-17T12:11:51.493690+0000 front 2024-09-17T12:11:51.493670+0000 (oldest deadline 2024-09-17T12:12:16.793582+0000) 2024-09-17T12:12:24.631 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:24 smithi165 podman[68909]: 2024-09-17 12:12:24.336683761 +0000 UTC m=+5.473458780 container died 12ba3cdf09dd682cc7b2eab26abc4f76166fd7e25861ff8de5f2ff84f4273438 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=main-7498493) 2024-09-17T12:12:24.632 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:24 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:24.355+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:24.632 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:24 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:24.355+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:24.632 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:24 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:24.355+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6822 osd.2 since back 2024-09-17T12:11:51.493690+0000 front 2024-09-17T12:11:51.493670+0000 (oldest deadline 2024-09-17T12:12:16.793582+0000) 2024-09-17T12:12:24.632 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:24 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:24.355+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6830 osd.3 since back 2024-09-17T12:11:59.094458+0000 front 2024-09-17T12:11:59.094468+0000 (oldest deadline 2024-09-17T12:12:23.794360+0000) 2024-09-17T12:12:24.957 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:24 smithi165 podman[68909]: 2024-09-17 12:12:24.730413307 +0000 UTC m=+5.867188321 container cleanup 12ba3cdf09dd682cc7b2eab26abc4f76166fd7e25861ff8de5f2ff84f4273438 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=main-7498493, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-17T12:12:24.957 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:24 smithi165 bash[68909]: ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6 2024-09-17T12:12:25.265 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:24 smithi165 podman[68942]: 2024-09-17 12:12:24.956615284 +0000 UTC m=+0.617448219 container remove 12ba3cdf09dd682cc7b2eab26abc4f76166fd7e25861ff8de5f2ff84f4273438 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=main-7498493, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T12:12:25.538 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:25 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:25.386+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:25.538 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:25 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:25.386+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:25.538 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:25 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:25.386+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6822 osd.2 since back 2024-09-17T12:11:51.493690+0000 front 2024-09-17T12:11:51.493670+0000 (oldest deadline 2024-09-17T12:12:16.793582+0000) 2024-09-17T12:12:25.538 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:25 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:25.386+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6830 osd.3 since back 2024-09-17T12:11:59.094458+0000 front 2024-09-17T12:11:59.094468+0000 (oldest deadline 2024-09-17T12:12:23.794360+0000) 2024-09-17T12:12:25.539 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:25 smithi165 podman[69032]: 2024-09-17 12:12:25.188399024 +0000 UTC m=+0.015730051 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:12:25.539 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:25 smithi165 podman[69032]: 2024-09-17 12:12:25.290599105 +0000 UTC m=+0.117930123 container create 6694817ee8798ac3a78fb2a18e3021e5c1233f68a5002af03f4fe0667194c7f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=main-7498493, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-17T12:12:25.539 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:25 smithi165 podman[69032]: 2024-09-17 12:12:25.534409742 +0000 UTC m=+0.361740765 container init 6694817ee8798ac3a78fb2a18e3021e5c1233f68a5002af03f4fe0667194c7f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-deactivate, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=main-7498493, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-17T12:12:25.539 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:25 smithi165 podman[69032]: 2024-09-17 12:12:25.537827418 +0000 UTC m=+0.365158436 container start 6694817ee8798ac3a78fb2a18e3021e5c1233f68a5002af03f4fe0667194c7f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-deactivate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, RELEASE=main-7498493, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-17T12:12:26.010 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:25 smithi165 podman[69032]: 2024-09-17 12:12:25.634648061 +0000 UTC m=+0.461979097 container attach 6694817ee8798ac3a78fb2a18e3021e5c1233f68a5002af03f4fe0667194c7f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-deactivate, RELEASE=main-7498493, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T12:12:26.010 INFO:journalctl@ceph.osd.6.smithi165.stdout:Sep 17 12:12:25 smithi165 podman[69032]: 2024-09-17 12:12:25.724805458 +0000 UTC m=+0.552136492 container died 6694817ee8798ac3a78fb2a18e3021e5c1233f68a5002af03f4fe0667194c7f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-6-deactivate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T12:12:26.395 DEBUG:teuthology.orchestra.run.smithi165:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.6.service' 2024-09-17T12:12:26.440 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T12:12:26.440 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-17T12:12:26.440 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-17T12:12:26.441 DEBUG:teuthology.orchestra.run.smithi165:> sudo systemctl stop ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.7 2024-09-17T12:12:26.718 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:26 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:26.381+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:26.718 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:26 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:26.381+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:26.718 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:26 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:26.381+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6822 osd.2 since back 2024-09-17T12:11:51.493690+0000 front 2024-09-17T12:11:51.493670+0000 (oldest deadline 2024-09-17T12:12:16.793582+0000) 2024-09-17T12:12:26.718 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:26 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:26.381+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6830 osd.3 since back 2024-09-17T12:11:59.094458+0000 front 2024-09-17T12:11:59.094468+0000 (oldest deadline 2024-09-17T12:12:23.794360+0000) 2024-09-17T12:12:26.718 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:26 smithi165 systemd[1]: Stopping Ceph osd.7 for a12537de-74ea-11ef-bceb-c7b262605968... 2024-09-17T12:12:27.015 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:26 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:26.715+0000 7f64d2750640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T12:12:27.015 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:26 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:26.715+0000 7f64d2750640 -1 osd.7 152 *** Got signal Terminated *** 2024-09-17T12:12:27.015 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:26 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:26.715+0000 7f64d2750640 -1 osd.7 152 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T12:12:27.765 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:27 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:27.424+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:27.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:27 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:27.424+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:27.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:27 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:27.424+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6822 osd.2 since back 2024-09-17T12:11:51.493690+0000 front 2024-09-17T12:11:51.493670+0000 (oldest deadline 2024-09-17T12:12:16.793582+0000) 2024-09-17T12:12:27.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:27 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:27.424+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6830 osd.3 since back 2024-09-17T12:11:59.094458+0000 front 2024-09-17T12:11:59.094468+0000 (oldest deadline 2024-09-17T12:12:23.794360+0000) 2024-09-17T12:12:28.765 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:28 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:28.460+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:28.765 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:28 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:28.460+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:28.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:28 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:28.460+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6822 osd.2 since back 2024-09-17T12:11:51.493690+0000 front 2024-09-17T12:11:51.493670+0000 (oldest deadline 2024-09-17T12:12:16.793582+0000) 2024-09-17T12:12:28.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:28 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:28.460+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6830 osd.3 since back 2024-09-17T12:11:59.094458+0000 front 2024-09-17T12:11:59.094468+0000 (oldest deadline 2024-09-17T12:12:23.794360+0000) 2024-09-17T12:12:29.765 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:29 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:29.468+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:29.765 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:29 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:29.468+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:29.765 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:29 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:29.468+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6822 osd.2 since back 2024-09-17T12:11:51.493690+0000 front 2024-09-17T12:11:51.493670+0000 (oldest deadline 2024-09-17T12:12:16.793582+0000) 2024-09-17T12:12:29.765 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:29 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:29.468+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6830 osd.3 since back 2024-09-17T12:11:59.094458+0000 front 2024-09-17T12:11:59.094468+0000 (oldest deadline 2024-09-17T12:12:23.794360+0000) 2024-09-17T12:12:30.765 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:30 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:30.475+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:30.765 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:30 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:30.475+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:30.765 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:30 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:30.475+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6822 osd.2 since back 2024-09-17T12:11:51.493690+0000 front 2024-09-17T12:11:51.493670+0000 (oldest deadline 2024-09-17T12:12:16.793582+0000) 2024-09-17T12:12:30.765 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:30 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:30.475+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6830 osd.3 since back 2024-09-17T12:11:59.094458+0000 front 2024-09-17T12:11:59.094468+0000 (oldest deadline 2024-09-17T12:12:23.794360+0000) 2024-09-17T12:12:31.765 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:31 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:31.458+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6806 osd.0 since back 2024-09-17T12:11:32.791662+0000 front 2024-09-17T12:11:32.791794+0000 (oldest deadline 2024-09-17T12:11:57.491567+0000) 2024-09-17T12:12:31.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:31 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:31.458+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6814 osd.1 since back 2024-09-17T12:11:42.693007+0000 front 2024-09-17T12:11:42.692958+0000 (oldest deadline 2024-09-17T12:12:05.592801+0000) 2024-09-17T12:12:31.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:31 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:31.458+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6822 osd.2 since back 2024-09-17T12:11:51.493690+0000 front 2024-09-17T12:11:51.493670+0000 (oldest deadline 2024-09-17T12:12:16.793582+0000) 2024-09-17T12:12:31.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:31 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:31.458+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.45:6830 osd.3 since back 2024-09-17T12:11:59.094458+0000 front 2024-09-17T12:11:59.094468+0000 (oldest deadline 2024-09-17T12:12:23.794360+0000) 2024-09-17T12:12:31.766 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:31 smithi165 ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7[59290]: 2024-09-17T12:12:31.458+0000 7f64ced65640 -1 osd.7 152 heartbeat_check: no reply from 172.21.15.165:6804 osd.4 since back 2024-09-17T12:12:06.595459+0000 front 2024-09-17T12:12:06.595369+0000 (oldest deadline 2024-09-17T12:12:30.695404+0000) 2024-09-17T12:12:32.534 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:32 smithi165 podman[69135]: 2024-09-17 12:12:32.280057034 +0000 UTC m=+5.655649897 container died b0f172112f9e76f3ac7a37744e139dfa31edbfe415627209666ab9dcb4e9cc12 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=main-7498493) 2024-09-17T12:12:32.892 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:32 smithi165 podman[69135]: 2024-09-17 12:12:32.707237301 +0000 UTC m=+6.082830161 container cleanup b0f172112f9e76f3ac7a37744e139dfa31edbfe415627209666ab9dcb4e9cc12 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=main-7498493, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-17T12:12:32.892 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:32 smithi165 bash[69135]: ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7 2024-09-17T12:12:33.265 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:32 smithi165 podman[69163]: 2024-09-17 12:12:32.891724344 +0000 UTC m=+0.614038803 container remove b0f172112f9e76f3ac7a37744e139dfa31edbfe415627209666ab9dcb4e9cc12 (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=main-7498493) 2024-09-17T12:12:33.637 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:33 smithi165 podman[69252]: 2024-09-17 12:12:33.227938448 +0000 UTC m=+0.016090781 image pull 6c1fcab85602ba9d6ddde3c89adee6ee0382b7527a8ed3ee372c969068775c76 quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb 2024-09-17T12:12:33.637 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:33 smithi165 podman[69252]: 2024-09-17 12:12:33.334299537 +0000 UTC m=+0.122451869 container create 9af5691dd24c6db581b7c1cf1af927f67836e9f4efd2d99fde428708c8854f1c (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=main-7498493, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-17T12:12:33.637 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:33 smithi165 podman[69252]: 2024-09-17 12:12:33.553396771 +0000 UTC m=+0.341549103 container init 9af5691dd24c6db581b7c1cf1af927f67836e9f4efd2d99fde428708c8854f1c (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , RELEASE=main-7498493, CEPH_POINT_RELEASE=) 2024-09-17T12:12:33.637 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:33 smithi165 podman[69252]: 2024-09-17 12:12:33.556715536 +0000 UTC m=+0.344867878 container start 9af5691dd24c6db581b7c1cf1af927f67836e9f4efd2d99fde428708c8854f1c (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-deactivate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=main-7498493, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-17T12:12:33.637 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:33 smithi165 podman[69252]: 2024-09-17 12:12:33.63636733 +0000 UTC m=+0.424519680 container attach 9af5691dd24c6db581b7c1cf1af927f67836e9f4efd2d99fde428708c8854f1c (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=main-7498493, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-17T12:12:34.015 INFO:journalctl@ceph.osd.7.smithi165.stdout:Sep 17 12:12:33 smithi165 podman[69252]: 2024-09-17 12:12:33.755973501 +0000 UTC m=+0.544125836 container died 9af5691dd24c6db581b7c1cf1af927f67836e9f4efd2d99fde428708c8854f1c (image=quay.ceph.io/ceph-ci/ceph@sha256:d1ee255599fffe6c832aeedeed0b06c298863896564017e014e180f38b9ae1bb, name=ceph-a12537de-74ea-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=main-7498493, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-17T12:12:34.484 DEBUG:teuthology.orchestra.run.smithi165:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a12537de-74ea-11ef-bceb-c7b262605968@osd.7.service' 2024-09-17T12:12:34.525 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T12:12:34.525 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-17T12:12:34.526 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid a12537de-74ea-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-17T12:15:35.622 DEBUG:teuthology.orchestra.run.smithi165:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid a12537de-74ea-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-17T12:15:57.135 DEBUG:teuthology.orchestra.run.smithi045:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T12:15:57.172 DEBUG:teuthology.orchestra.run.smithi165:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T12:15:57.200 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-17T12:15:57.202 DEBUG:teuthology.misc:Transferring archived files from smithi045:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-16_20:08:15-orch-main-distro-default-smithi/7907794/remote/smithi045/crash 2024-09-17T12:15:57.203 DEBUG:teuthology.orchestra.run.smithi045:> sudo tar c -f - -C /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/crash -- . 2024-09-17T12:15:57.256 INFO:teuthology.orchestra.run.smithi045.stderr:tar: /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-17T12:15:57.256 INFO:teuthology.orchestra.run.smithi045.stderr:tar: Error is not recoverable: exiting now 2024-09-17T12:15:57.258 DEBUG:teuthology.misc:Transferring archived files from smithi165:/var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-16_20:08:15-orch-main-distro-default-smithi/7907794/remote/smithi165/crash 2024-09-17T12:15:57.259 DEBUG:teuthology.orchestra.run.smithi165:> sudo tar c -f - -C /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/crash -- . 2024-09-17T12:15:57.295 INFO:teuthology.orchestra.run.smithi165.stderr:tar: /var/lib/ceph/a12537de-74ea-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-17T12:15:57.295 INFO:teuthology.orchestra.run.smithi165.stderr:tar: Error is not recoverable: exiting now 2024-09-17T12:15:57.296 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-17T12:15:57.297 DEBUG:teuthology.orchestra.run.smithi045:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_REFRESH_FAILED | head -n 1 2024-09-17T12:15:57.331 INFO:tasks.cephadm:Compressing logs... 2024-09-17T12:15:57.331 DEBUG:teuthology.orchestra.run.smithi045:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-17T12:15:57.374 DEBUG:teuthology.orchestra.run.smithi165:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-17T12:15:57.399 INFO:teuthology.orchestra.run.smithi165.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-17T12:15:57.400 INFO:teuthology.orchestra.run.smithi165.stderr:: No such file or directory 2024-09-17T12:15:57.400 INFO:teuthology.orchestra.run.smithi165.stderr:gzip -5 --verbose -- /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-17T12:15:57.400 INFO:teuthology.orchestra.run.smithi165.stderr:/var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-17T12:15:57.400 INFO:teuthology.orchestra.run.smithi165.stderr:/var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-17T12:15:57.403 INFO:teuthology.orchestra.run.smithi165.stderr:/var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-mon.smithi165.log 2024-09-17T12:15:57.403 INFO:teuthology.orchestra.run.smithi165.stderr:gzip -5 --verbose -- /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-17T12:15:57.403 INFO:teuthology.orchestra.run.smithi165.stderr:/var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph.audit.log: /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-mon.smithi165.log: gzip -5 --verbose -- /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi165.czpmgd.log 2024-09-17T12:15:57.403 INFO:teuthology.orchestra.run.smithi165.stderr:/var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-mgr.smithi165.ykdszc.log 2024-09-17T12:15:57.403 INFO:teuthology.orchestra.run.smithi165.stderr:/var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi165.czpmgd.log: gzip -5 --verbose -- /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-17T12:15:57.403 INFO:teuthology.orchestra.run.smithi165.stderr: 87.8% -- replaced with /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi165.czpmgd.log.gz 2024-09-17T12:15:57.403 INFO:teuthology.orchestra.run.smithi165.stderr: 85.7% -- replaced with /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-17T12:15:57.403 INFO:teuthology.orchestra.run.smithi165.stderr:/var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-mgr.smithi165.ykdszc.log: gzip -5 --verbose -- /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-17T12:15:57.406 INFO:teuthology.orchestra.run.smithi045.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-17T12:15:57.406 INFO:teuthology.orchestra.run.smithi045.stderr:: No such file or directory 2024-09-17T12:15:57.406 INFO:teuthology.orchestra.run.smithi045.stderr:gzip -5 --verbose -- /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-17T12:15:57.406 INFO:teuthology.orchestra.run.smithi045.stderr:gzip -5 --verbose -- /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi045.olwaas.log 2024-09-17T12:15:57.406 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.0.log: /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-mgr.smithi045.jtwyrp.log 2024-09-17T12:15:57.406 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi045.olwaas.log: gzip -5 --verbose -- /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-17T12:15:57.406 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-mgr.smithi045.jtwyrp.log: gzip -5 --verbose -- /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-17T12:15:57.406 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-17T12:15:57.407 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-mon.smithi045.log 2024-09-17T12:15:57.407 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-17T12:15:57.407 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-mon.smithi045.log: 85.8% -- replaced with /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-17T12:15:57.407 INFO:teuthology.orchestra.run.smithi045.stderr:gzip -5 --verbose -- /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph.log 2024-09-17T12:15:57.407 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.2.log: 91.4% -- replaced with /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-17T12:15:57.408 INFO:teuthology.orchestra.run.smithi045.stderr:gzip -5 --verbose -- /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-17T12:15:57.408 INFO:teuthology.orchestra.run.smithi165.stderr:/var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.6.log: 91.6% -- replaced with /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-17T12:15:57.408 INFO:teuthology.orchestra.run.smithi165.stderr:gzip -5 --verbose -- /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph.log 2024-09-17T12:15:57.408 INFO:teuthology.orchestra.run.smithi165.stderr:/var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-17T12:15:57.408 INFO:teuthology.orchestra.run.smithi165.stderr:/var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph.log: 87.0% -- replaced with /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-17T12:15:57.411 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph.log: 87.0% -- replaced with /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-17T12:15:57.411 INFO:teuthology.orchestra.run.smithi045.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-17T12:15:57.413 INFO:teuthology.orchestra.run.smithi165.stderr: 91.2% -- replaced with /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-17T12:15:57.416 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.3.log: 91.1% -- replaced with /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-17T12:15:57.425 INFO:teuthology.orchestra.run.smithi045.stderr:/var/log/ceph/cephadm.log: 91.2% -- replaced with /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi045.olwaas.log.gz 2024-09-17T12:15:57.428 INFO:teuthology.orchestra.run.smithi165.stderr:/var/log/ceph/cephadm.log: 90.0% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-17T12:15:57.446 INFO:teuthology.orchestra.run.smithi045.stderr: 91.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-17T12:15:57.471 INFO:teuthology.orchestra.run.smithi165.stderr: 89.8% -- replaced with /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-mgr.smithi165.ykdszc.log.gz 2024-09-17T12:15:57.737 INFO:teuthology.orchestra.run.smithi165.stderr: 92.6% -- replaced with /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-mon.smithi165.log.gz 2024-09-17T12:15:57.789 INFO:teuthology.orchestra.run.smithi045.stderr: 88.6% -- replaced with /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-mgr.smithi045.jtwyrp.log.gz 2024-09-17T12:15:58.733 INFO:teuthology.orchestra.run.smithi045.stderr: 91.8% -- replaced with /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-mon.smithi045.log.gz 2024-09-17T12:16:09.482 INFO:teuthology.orchestra.run.smithi045.stderr: 94.5% -- replaced with /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-17T12:16:10.647 INFO:teuthology.orchestra.run.smithi165.stderr: 93.9% -- replaced with /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-17T12:16:11.812 INFO:teuthology.orchestra.run.smithi165.stderr: 94.2% -- replaced with /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-17T12:16:13.922 INFO:teuthology.orchestra.run.smithi045.stderr: 94.1% -- replaced with /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-17T12:16:14.571 INFO:teuthology.orchestra.run.smithi045.stderr: 94.2% -- replaced with /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-17T12:16:15.251 INFO:teuthology.orchestra.run.smithi045.stderr: 94.1% -- replaced with /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-17T12:16:15.253 INFO:teuthology.orchestra.run.smithi045.stderr: 2024-09-17T12:16:15.253 INFO:teuthology.orchestra.run.smithi045.stderr:real 0m17.865s 2024-09-17T12:16:15.253 INFO:teuthology.orchestra.run.smithi045.stderr:user 1m3.056s 2024-09-17T12:16:15.253 INFO:teuthology.orchestra.run.smithi045.stderr:sys 0m2.255s 2024-09-17T12:16:16.027 INFO:teuthology.orchestra.run.smithi165.stderr: 94.2% -- replaced with /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-17T12:16:20.810 INFO:teuthology.orchestra.run.smithi165.stderr: 94.1% -- replaced with /var/log/ceph/a12537de-74ea-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-17T12:16:20.812 INFO:teuthology.orchestra.run.smithi165.stderr: 2024-09-17T12:16:20.812 INFO:teuthology.orchestra.run.smithi165.stderr:real 0m23.423s 2024-09-17T12:16:20.812 INFO:teuthology.orchestra.run.smithi165.stderr:user 1m7.306s 2024-09-17T12:16:20.812 INFO:teuthology.orchestra.run.smithi165.stderr:sys 0m2.686s 2024-09-17T12:16:20.813 INFO:tasks.cephadm:Archiving logs... 2024-09-17T12:16:20.813 DEBUG:teuthology.misc:Transferring archived files from smithi045:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-16_20:08:15-orch-main-distro-default-smithi/7907794/remote/smithi045/log 2024-09-17T12:16:20.814 DEBUG:teuthology.orchestra.run.smithi045:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-17T12:16:29.354 DEBUG:teuthology.misc:Transferring archived files from smithi165:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-16_20:08:15-orch-main-distro-default-smithi/7907794/remote/smithi165/log 2024-09-17T12:16:29.355 DEBUG:teuthology.orchestra.run.smithi165:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-17T12:16:39.008 INFO:tasks.cephadm:Removing cluster... 2024-09-17T12:16:39.008 DEBUG:teuthology.orchestra.run.smithi045:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid a12537de-74ea-11ef-bceb-c7b262605968 --force 2024-09-17T12:16:39.377 DEBUG:teuthology.orchestra.run.smithi165:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid a12537de-74ea-11ef-bceb-c7b262605968 --force 2024-09-17T12:16:39.755 INFO:tasks.cephadm:Removing cephadm ... 2024-09-17T12:16:39.755 DEBUG:teuthology.orchestra.run.smithi045:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-17T12:16:39.773 DEBUG:teuthology.orchestra.run.smithi165:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-17T12:16:39.788 INFO:tasks.cephadm:Teardown complete 2024-09-17T12:16:39.788 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-17T12:16:39.799 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-17T12:16:39.799 DEBUG:teuthology.orchestra.run.smithi045:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T12:16:39.816 DEBUG:teuthology.orchestra.run.smithi165:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T12:16:39.829 INFO:teuthology.orchestra.run.smithi045.stderr:bash: line 1: ntpq: command not found 2024-09-17T12:16:39.833 INFO:teuthology.orchestra.run.smithi045.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T12:16:39.833 INFO:teuthology.orchestra.run.smithi045.stdout:=============================================================================== 2024-09-17T12:16:39.833 INFO:teuthology.orchestra.run.smithi045.stdout:^* hv01.front.sepia.ceph.com 3 6 377 58 -611us[ -539us] +/- 60ms 2024-09-17T12:16:39.834 INFO:teuthology.orchestra.run.smithi045.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 55 +1848us[+1848us] +/- 58ms 2024-09-17T12:16:39.834 INFO:teuthology.orchestra.run.smithi045.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 53 -1213us[-1213us] +/- 70ms 2024-09-17T12:16:39.834 INFO:teuthology.orchestra.run.smithi045.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T12:16:39.841 INFO:teuthology.orchestra.run.smithi165.stderr:bash: line 1: ntpq: command not found 2024-09-17T12:16:39.846 INFO:teuthology.orchestra.run.smithi165.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T12:16:39.847 INFO:teuthology.orchestra.run.smithi165.stdout:=============================================================================== 2024-09-17T12:16:39.847 INFO:teuthology.orchestra.run.smithi165.stdout:^* hv01.front.sepia.ceph.com 3 6 377 57 -626us[ -563us] +/- 60ms 2024-09-17T12:16:39.847 INFO:teuthology.orchestra.run.smithi165.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 55 +1802us[+1802us] +/- 58ms 2024-09-17T12:16:39.847 INFO:teuthology.orchestra.run.smithi165.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 56 -1247us[-1247us] +/- 70ms 2024-09-17T12:16:39.847 INFO:teuthology.orchestra.run.smithi165.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T12:16:39.847 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-17T12:16:39.860 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-17T12:16:39.860 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-17T12:16:39.875 DEBUG:teuthology.orchestra.run.smithi045:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-17T12:16:39.955 DEBUG:teuthology.orchestra.run.smithi165:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-17T12:16:40.026 DEBUG:teuthology.orchestra.run.smithi045:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-17T12:16:40.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T12:16:40.055 DEBUG:teuthology.orchestra.run.smithi165:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-17T12:16:40.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T12:16:40.083 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-17T12:16:40.094 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-17T12:16:40.181 INFO:teuthology.task.internal:Duration was 1976.700234 seconds 2024-09-17T12:16:40.182 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-17T12:16:40.194 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-17T12:16:40.195 DEBUG:teuthology.orchestra.run.smithi045:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-17T12:16:40.197 DEBUG:teuthology.orchestra.run.smithi165:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-17T12:16:40.235 INFO:teuthology.orchestra.run.smithi045.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T12:16:40.236 INFO:teuthology.orchestra.run.smithi165.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T12:16:40.630 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-17T12:16:40.630 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi045.front.sepia.ceph.com 2024-09-17T12:16:40.630 DEBUG:teuthology.orchestra.run.smithi045:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-17T12:16:40.693 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi165.front.sepia.ceph.com 2024-09-17T12:16:40.693 DEBUG:teuthology.orchestra.run.smithi165:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-17T12:16:40.743 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-17T12:16:40.743 DEBUG:teuthology.orchestra.run.smithi045:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-17T12:16:40.746 DEBUG:teuthology.orchestra.run.smithi165:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-17T12:16:40.772 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-17T12:16:40.772 DEBUG:teuthology.orchestra.run.smithi045:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-17T12:16:40.815 DEBUG:teuthology.orchestra.run.smithi165:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-17T12:16:55.016 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-17T12:16:55.029 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-17T12:16:55.029 DEBUG:teuthology.orchestra.run.smithi045:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-17T12:16:55.055 DEBUG:teuthology.orchestra.run.smithi165:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-17T12:16:55.087 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-17T12:16:55.102 DEBUG:teuthology.orchestra.run.smithi045:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-17T12:16:55.104 DEBUG:teuthology.orchestra.run.smithi165:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-17T12:16:55.126 INFO:teuthology.orchestra.run.smithi045.stdout:kernel.core_pattern = core 2024-09-17T12:16:55.155 INFO:teuthology.orchestra.run.smithi165.stdout:kernel.core_pattern = core 2024-09-17T12:16:55.206 DEBUG:teuthology.orchestra.run.smithi045:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-17T12:16:55.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T12:16:55.222 DEBUG:teuthology.orchestra.run.smithi165:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-17T12:16:55.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T12:16:55.264 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-17T12:16:55.274 INFO:teuthology.task.internal:Transferring archived files... 2024-09-17T12:16:55.276 DEBUG:teuthology.misc:Transferring archived files from smithi045:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-16_20:08:15-orch-main-distro-default-smithi/7907794/remote/smithi045 2024-09-17T12:16:55.276 DEBUG:teuthology.orchestra.run.smithi045:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-17T12:16:55.344 DEBUG:teuthology.misc:Transferring archived files from smithi165:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-16_20:08:15-orch-main-distro-default-smithi/7907794/remote/smithi165 2024-09-17T12:16:55.344 DEBUG:teuthology.orchestra.run.smithi165:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-17T12:16:55.386 INFO:teuthology.task.internal:Removing archive directory... 2024-09-17T12:16:55.386 DEBUG:teuthology.orchestra.run.smithi045:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-17T12:16:55.388 DEBUG:teuthology.orchestra.run.smithi165:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-17T12:16:55.436 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-17T12:16:55.447 INFO:teuthology.task.internal:Not uploading archives. 2024-09-17T12:16:55.448 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-17T12:16:55.556 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-17T12:16:55.556 DEBUG:teuthology.orchestra.run.smithi045:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-17T12:16:55.558 DEBUG:teuthology.orchestra.run.smithi165:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-17T12:16:55.572 INFO:teuthology.orchestra.run.smithi045.stdout: 265324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 17 12:16 /home/ubuntu/cephtest 2024-09-17T12:16:55.574 INFO:teuthology.orchestra.run.smithi165.stdout: 266064 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 17 12:16 /home/ubuntu/cephtest 2024-09-17T12:16:55.575 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-17T12:16:55.586 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-17T12:16:55.613 INFO:teuthology.run:Summary data: description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 1976.700234413147 owner: scheduled_teuthology@teuthology success: true 2024-09-17T12:16:55.613 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-17T12:16:55.701 INFO:teuthology.run:pass